From 25464456d38519a646256cdcabbedfa25b0a1be5 Mon Sep 17 00:00:00 2001 From: Tyler Benson Date: Fri, 13 Mar 2020 14:52:12 -0700 Subject: [PATCH 01/43] Add tests for http clients - dropped request - non-routable request - https request Unfortunately I wasn't able to figure out a clean way to test some of these scenarios without making a remote request. I was also not able to configure everything consistently, so I had to disable that test for several integrations. --- .../AkkaHttpClientInstrumentationTest.groovy | 8 ++ .../ApacheHttpAsyncClientCallbackTest.groovy | 11 ++- ...acheHttpAsyncClientNullCallbackTest.groovy | 11 ++- .../groovy/ApacheHttpAsyncClientTest.groovy | 11 ++- ...ApacheHttpClientResponseHandlerTest.groovy | 10 +++ .../test/groovy/ApacheHttpClientTest.groovy | 37 ++++++++ .../test/groovy/CommonsHttpClientTest.groovy | 7 ++ .../AbstractGoogleHttpClientTest.groovy | 2 + .../groovy/GoogleHttpClientAsyncTest.groovy | 2 + .../test/groovy/GoogleHttpClientTest.groovy | 5 +- ...tpUrlConnectionResponseCodeOnlyTest.groovy | 4 + .../test/groovy/HttpUrlConnectionTest.groovy | 4 + ...HttpUrlConnectionUseCachesFalseTest.groovy | 4 + .../test/groovy/SpringRestTemplateTest.groovy | 19 +++- .../src/test/groovy/JaxRsClientV1Test.groovy | 2 + .../jax-rs-client-2.0.gradle | 3 +- .../test/groovy/JaxRsClientAsyncTest.groovy | 19 +++- .../src/test/groovy/JaxRsClientTest.groovy | 22 ++++- .../src/test/groovy/Netty40ClientTest.groovy | 7 ++ .../src/test/groovy/Netty41ClientTest.groovy | 7 ++ .../src/test/groovy/OkHttp3Test.groovy | 10 ++- .../groovy/client/PlayWSClientTest.groovy | 7 ++ .../src/test/groovy/PlayWSClientTest.groovy | 5 ++ .../client/RatpackHttpClientTest.groovy | 15 +++- .../client/SpringWebfluxHttpClientTest.groovy | 7 ++ .../groovy/client/VertxHttpClientTest.groovy | 13 ++- .../VertxRxCircuitBreakerWebClientTest.groovy | 10 ++- .../groovy/client/VertxRxWebClientTest.groovy | 10 ++- .../agent/test/base/HttpClientTest.groovy | 88 +++++++++++++++++-- .../test/server/http/TestHttpServer.groovy | 7 +- .../src/test/groovy/server/ServerTest.groovy | 7 +- 31 files changed, 340 insertions(+), 34 deletions(-) diff --git a/dd-java-agent/instrumentation/akka-http-10.0/src/test/groovy/AkkaHttpClientInstrumentationTest.groovy b/dd-java-agent/instrumentation/akka-http-10.0/src/test/groovy/AkkaHttpClientInstrumentationTest.groovy index 6dffff7f1f..0e04cbe190 100644 --- a/dd-java-agent/instrumentation/akka-http-10.0/src/test/groovy/AkkaHttpClientInstrumentationTest.groovy +++ b/dd-java-agent/instrumentation/akka-http-10.0/src/test/groovy/AkkaHttpClientInstrumentationTest.groovy @@ -9,7 +9,9 @@ import datadog.trace.api.DDSpanTypes import datadog.trace.bootstrap.instrumentation.api.Tags import datadog.trace.instrumentation.akkahttp.AkkaHttpClientDecorator import spock.lang.Shared +import spock.lang.Timeout +@Timeout(5) class AkkaHttpClientInstrumentationTest extends HttpClientTest { @Shared @@ -56,6 +58,12 @@ class AkkaHttpClientInstrumentationTest extends HttpClientTest { false } + @Override + boolean testRemoteConnection() { + // Not sure how to properly set timeouts... + return false + } + def "singleRequest exception trace"() { when: // Passing null causes NPE in singleRequest diff --git a/dd-java-agent/instrumentation/apache-httpasyncclient-4/src/test/groovy/ApacheHttpAsyncClientCallbackTest.groovy b/dd-java-agent/instrumentation/apache-httpasyncclient-4/src/test/groovy/ApacheHttpAsyncClientCallbackTest.groovy index 4f7620326d..d5248f6522 100644 --- a/dd-java-agent/instrumentation/apache-httpasyncclient-4/src/test/groovy/ApacheHttpAsyncClientCallbackTest.groovy +++ b/dd-java-agent/instrumentation/apache-httpasyncclient-4/src/test/groovy/ApacheHttpAsyncClientCallbackTest.groovy @@ -1,19 +1,28 @@ import datadog.trace.agent.test.base.HttpClientTest import datadog.trace.instrumentation.apachehttpasyncclient.ApacheHttpAsyncClientDecorator import org.apache.http.HttpResponse +import org.apache.http.client.config.RequestConfig import org.apache.http.concurrent.FutureCallback import org.apache.http.impl.nio.client.HttpAsyncClients import org.apache.http.message.BasicHeader import spock.lang.AutoCleanup import spock.lang.Shared +import spock.lang.Timeout import java.util.concurrent.CompletableFuture +@Timeout(5) class ApacheHttpAsyncClientCallbackTest extends HttpClientTest { + @Shared + RequestConfig requestConfig = RequestConfig.custom() + .setSocketTimeout(1000) + .setConnectTimeout(1000) + .setConnectionRequestTimeout(1000).build() + @AutoCleanup @Shared - def client = HttpAsyncClients.createDefault() + def client = HttpAsyncClients.custom().setDefaultRequestConfig(requestConfig).build() def setupSpec() { client.start() diff --git a/dd-java-agent/instrumentation/apache-httpasyncclient-4/src/test/groovy/ApacheHttpAsyncClientNullCallbackTest.groovy b/dd-java-agent/instrumentation/apache-httpasyncclient-4/src/test/groovy/ApacheHttpAsyncClientNullCallbackTest.groovy index da7d589f94..4cf58232d9 100644 --- a/dd-java-agent/instrumentation/apache-httpasyncclient-4/src/test/groovy/ApacheHttpAsyncClientNullCallbackTest.groovy +++ b/dd-java-agent/instrumentation/apache-httpasyncclient-4/src/test/groovy/ApacheHttpAsyncClientNullCallbackTest.groovy @@ -1,17 +1,26 @@ import datadog.trace.agent.test.base.HttpClientTest import datadog.trace.instrumentation.apachehttpasyncclient.ApacheHttpAsyncClientDecorator +import org.apache.http.client.config.RequestConfig import org.apache.http.impl.nio.client.HttpAsyncClients import org.apache.http.message.BasicHeader import spock.lang.AutoCleanup import spock.lang.Shared +import spock.lang.Timeout import java.util.concurrent.Future +@Timeout(5) class ApacheHttpAsyncClientNullCallbackTest extends HttpClientTest { + @Shared + RequestConfig requestConfig = RequestConfig.custom() + .setSocketTimeout(1000) + .setConnectTimeout(1000) + .setConnectionRequestTimeout(1000).build() + @AutoCleanup @Shared - def client = HttpAsyncClients.createDefault() + def client = HttpAsyncClients.custom().setDefaultRequestConfig(requestConfig).build() def setupSpec() { client.start() diff --git a/dd-java-agent/instrumentation/apache-httpasyncclient-4/src/test/groovy/ApacheHttpAsyncClientTest.groovy b/dd-java-agent/instrumentation/apache-httpasyncclient-4/src/test/groovy/ApacheHttpAsyncClientTest.groovy index f4056ca483..228c2fbcba 100644 --- a/dd-java-agent/instrumentation/apache-httpasyncclient-4/src/test/groovy/ApacheHttpAsyncClientTest.groovy +++ b/dd-java-agent/instrumentation/apache-httpasyncclient-4/src/test/groovy/ApacheHttpAsyncClientTest.groovy @@ -1,17 +1,26 @@ import datadog.trace.agent.test.base.HttpClientTest import datadog.trace.instrumentation.apachehttpasyncclient.ApacheHttpAsyncClientDecorator import org.apache.http.HttpResponse +import org.apache.http.client.config.RequestConfig import org.apache.http.concurrent.FutureCallback import org.apache.http.impl.nio.client.HttpAsyncClients import org.apache.http.message.BasicHeader import spock.lang.AutoCleanup import spock.lang.Shared +import spock.lang.Timeout +@Timeout(5) class ApacheHttpAsyncClientTest extends HttpClientTest { + @Shared + RequestConfig requestConfig = RequestConfig.custom() + .setSocketTimeout(1000) + .setConnectTimeout(1000) + .setConnectionRequestTimeout(1000).build() + @AutoCleanup @Shared - def client = HttpAsyncClients.createDefault() + def client = HttpAsyncClients.custom().setDefaultRequestConfig(requestConfig).build() def setupSpec() { client.start() diff --git a/dd-java-agent/instrumentation/apache-httpclient-4/src/test/groovy/ApacheHttpClientResponseHandlerTest.groovy b/dd-java-agent/instrumentation/apache-httpclient-4/src/test/groovy/ApacheHttpClientResponseHandlerTest.groovy index ecc68310d9..b7a03b697c 100644 --- a/dd-java-agent/instrumentation/apache-httpclient-4/src/test/groovy/ApacheHttpClientResponseHandlerTest.groovy +++ b/dd-java-agent/instrumentation/apache-httpclient-4/src/test/groovy/ApacheHttpClientResponseHandlerTest.groovy @@ -4,8 +4,12 @@ import org.apache.http.HttpResponse import org.apache.http.client.ResponseHandler import org.apache.http.impl.client.DefaultHttpClient import org.apache.http.message.BasicHeader +import org.apache.http.params.HttpConnectionParams +import org.apache.http.params.HttpParams import spock.lang.Shared +import spock.lang.Timeout +@Timeout(5) class ApacheHttpClientResponseHandlerTest extends HttpClientTest { @Shared @@ -19,6 +23,12 @@ class ApacheHttpClientResponseHandlerTest extends HttpClientTest { } } + def setupSpec() { + HttpParams httpParams = client.getParams() + HttpConnectionParams.setConnectionTimeout(httpParams, 2000) + HttpConnectionParams.setSoTimeout(httpParams, 2000) + } + @Override int doRequest(String method, URI uri, Map headers, Closure callback) { def request = new HttpUriRequest(method, uri) diff --git a/dd-java-agent/instrumentation/apache-httpclient-4/src/test/groovy/ApacheHttpClientTest.groovy b/dd-java-agent/instrumentation/apache-httpclient-4/src/test/groovy/ApacheHttpClientTest.groovy index 23d5ba7dd2..49841e1c10 100644 --- a/dd-java-agent/instrumentation/apache-httpclient-4/src/test/groovy/ApacheHttpClientTest.groovy +++ b/dd-java-agent/instrumentation/apache-httpclient-4/src/test/groovy/ApacheHttpClientTest.groovy @@ -6,13 +6,22 @@ import org.apache.http.HttpResponse import org.apache.http.impl.client.DefaultHttpClient import org.apache.http.message.BasicHeader import org.apache.http.message.BasicHttpRequest +import org.apache.http.params.HttpConnectionParams +import org.apache.http.params.HttpParams import org.apache.http.protocol.BasicHttpContext import spock.lang.Shared +import spock.lang.Timeout abstract class ApacheHttpClientTest extends HttpClientTest { @Shared def client = new DefaultHttpClient() + def setupSpec() { + HttpParams httpParams = client.getParams() + HttpConnectionParams.setConnectionTimeout(httpParams, 2000) + HttpConnectionParams.setSoTimeout(httpParams, 2000) + } + @Override String component() { return ApacheHttpClientDecorator.DECORATE.component() @@ -55,6 +64,7 @@ abstract class ApacheHttpClientTest extends HttpClientTes } } +@Timeout(5) class ApacheClientHostRequest extends ApacheHttpClientTest { @Override BasicHttpRequest createRequest(String method, URI uri) { @@ -65,8 +75,14 @@ class ApacheClientHostRequest extends ApacheHttpClientTest { HttpResponse executeRequest(BasicHttpRequest request, URI uri) { return client.execute(new HttpHost(uri.getHost(), uri.getPort()), request) } + + @Override + boolean testRemoteConnection() { + return false + } } +@Timeout(5) class ApacheClientHostRequestContext extends ApacheHttpClientTest { @Override BasicHttpRequest createRequest(String method, URI uri) { @@ -77,8 +93,14 @@ class ApacheClientHostRequestContext extends ApacheHttpClientTest { @Override BasicHttpRequest createRequest(String method, URI uri) { @@ -89,8 +111,14 @@ class ApacheClientHostRequestResponseHandler extends ApacheHttpClientTest response }) } + + @Override + boolean testRemoteConnection() { + return false + } } +@Timeout(5) class ApacheClientHostRequestResponseHandlerContext extends ApacheHttpClientTest { @Override BasicHttpRequest createRequest(String method, URI uri) { @@ -101,8 +129,14 @@ class ApacheClientHostRequestResponseHandlerContext extends ApacheHttpClientTest HttpResponse executeRequest(BasicHttpRequest request, URI uri) { return client.execute(new HttpHost(uri.getHost(), uri.getPort()), request, { response -> response }, new BasicHttpContext()) } + + @Override + boolean testRemoteConnection() { + return false + } } +@Timeout(5) class ApacheClientUriRequest extends ApacheHttpClientTest { @Override HttpUriRequest createRequest(String method, URI uri) { @@ -115,6 +149,7 @@ class ApacheClientUriRequest extends ApacheHttpClientTest { } } +@Timeout(5) class ApacheClientUriRequestContext extends ApacheHttpClientTest { @Override HttpUriRequest createRequest(String method, URI uri) { @@ -127,6 +162,7 @@ class ApacheClientUriRequestContext extends ApacheHttpClientTest } } +@Timeout(5) class ApacheClientUriRequestResponseHandler extends ApacheHttpClientTest { @Override HttpUriRequest createRequest(String method, URI uri) { @@ -139,6 +175,7 @@ class ApacheClientUriRequestResponseHandler extends ApacheHttpClientTest { @Override HttpUriRequest createRequest(String method, URI uri) { diff --git a/dd-java-agent/instrumentation/commons-httpclient-2/src/test/groovy/CommonsHttpClientTest.groovy b/dd-java-agent/instrumentation/commons-httpclient-2/src/test/groovy/CommonsHttpClientTest.groovy index d482389909..dc1456d812 100644 --- a/dd-java-agent/instrumentation/commons-httpclient-2/src/test/groovy/CommonsHttpClientTest.groovy +++ b/dd-java-agent/instrumentation/commons-httpclient-2/src/test/groovy/CommonsHttpClientTest.groovy @@ -10,11 +10,18 @@ import org.apache.commons.httpclient.methods.PostMethod import org.apache.commons.httpclient.methods.PutMethod import org.apache.commons.httpclient.methods.TraceMethod import spock.lang.Shared +import spock.lang.Timeout +@Timeout(5) class CommonsHttpClientTest extends HttpClientTest { @Shared HttpClient client = new HttpClient() + def setupSpec() { + client.setConnectionTimeout(2000) + client.setTimeout(2000) + } + @Override int doRequest(String method, URI uri, Map headers, Closure callback) { HttpMethod httpMethod diff --git a/dd-java-agent/instrumentation/google-http-client/src/test/groovy/AbstractGoogleHttpClientTest.groovy b/dd-java-agent/instrumentation/google-http-client/src/test/groovy/AbstractGoogleHttpClientTest.groovy index 785a8f7cc8..0ffa54e67b 100644 --- a/dd-java-agent/instrumentation/google-http-client/src/test/groovy/AbstractGoogleHttpClientTest.groovy +++ b/dd-java-agent/instrumentation/google-http-client/src/test/groovy/AbstractGoogleHttpClientTest.groovy @@ -23,6 +23,8 @@ abstract class AbstractGoogleHttpClientTest extends HttpClientTest { GenericUrl genericUrl = new GenericUrl(uri) HttpRequest request = requestFactory.buildRequest(method, genericUrl, null) + request.connectTimeout = 2000 + request.readTimeout = 2000 request.getHeaders().putAll(headers) request.setThrowExceptionOnExecuteError(throwExceptionOnError) diff --git a/dd-java-agent/instrumentation/google-http-client/src/test/groovy/GoogleHttpClientAsyncTest.groovy b/dd-java-agent/instrumentation/google-http-client/src/test/groovy/GoogleHttpClientAsyncTest.groovy index 934e31dc54..73b279080c 100644 --- a/dd-java-agent/instrumentation/google-http-client/src/test/groovy/GoogleHttpClientAsyncTest.groovy +++ b/dd-java-agent/instrumentation/google-http-client/src/test/groovy/GoogleHttpClientAsyncTest.groovy @@ -1,6 +1,8 @@ import com.google.api.client.http.HttpRequest import com.google.api.client.http.HttpResponse +import spock.lang.Timeout +@Timeout(5) class GoogleHttpClientAsyncTest extends AbstractGoogleHttpClientTest { @Override HttpResponse executeRequest(HttpRequest request) { diff --git a/dd-java-agent/instrumentation/google-http-client/src/test/groovy/GoogleHttpClientTest.groovy b/dd-java-agent/instrumentation/google-http-client/src/test/groovy/GoogleHttpClientTest.groovy index 447264eeeb..ab1ba4a352 100644 --- a/dd-java-agent/instrumentation/google-http-client/src/test/groovy/GoogleHttpClientTest.groovy +++ b/dd-java-agent/instrumentation/google-http-client/src/test/groovy/GoogleHttpClientTest.groovy @@ -1,9 +1,12 @@ import com.google.api.client.http.HttpRequest import com.google.api.client.http.HttpResponse import spock.lang.Retry +import spock.lang.Timeout -@Retry +@Retry(condition = { !invocation.method.name.contains('circular redirects') }) +@Timeout(5) class GoogleHttpClientTest extends AbstractGoogleHttpClientTest { + @Override HttpResponse executeRequest(HttpRequest request) { return request.execute() diff --git a/dd-java-agent/instrumentation/http-url-connection/src/test/groovy/HttpUrlConnectionResponseCodeOnlyTest.groovy b/dd-java-agent/instrumentation/http-url-connection/src/test/groovy/HttpUrlConnectionResponseCodeOnlyTest.groovy index dc592caa8e..0b8d08dac0 100644 --- a/dd-java-agent/instrumentation/http-url-connection/src/test/groovy/HttpUrlConnectionResponseCodeOnlyTest.groovy +++ b/dd-java-agent/instrumentation/http-url-connection/src/test/groovy/HttpUrlConnectionResponseCodeOnlyTest.groovy @@ -1,6 +1,8 @@ import datadog.trace.agent.test.base.HttpClientTest import datadog.trace.instrumentation.http_url_connection.HttpUrlConnectionDecorator +import spock.lang.Timeout +@Timeout(5) class HttpUrlConnectionResponseCodeOnlyTest extends HttpClientTest { @Override @@ -8,6 +10,8 @@ class HttpUrlConnectionResponseCodeOnlyTest extends HttpClientTest { HttpURLConnection connection = uri.toURL().openConnection() try { connection.setRequestMethod(method) + connection.connectTimeout = 2000 + connection.readTimeout = 2000 headers.each { connection.setRequestProperty(it.key, it.value) } connection.setRequestProperty("Connection", "close") return connection.getResponseCode() diff --git a/dd-java-agent/instrumentation/http-url-connection/src/test/groovy/HttpUrlConnectionTest.groovy b/dd-java-agent/instrumentation/http-url-connection/src/test/groovy/HttpUrlConnectionTest.groovy index c4a3c7e1da..0da17b75d1 100644 --- a/dd-java-agent/instrumentation/http-url-connection/src/test/groovy/HttpUrlConnectionTest.groovy +++ b/dd-java-agent/instrumentation/http-url-connection/src/test/groovy/HttpUrlConnectionTest.groovy @@ -5,6 +5,7 @@ import datadog.trace.bootstrap.instrumentation.api.Tags import datadog.trace.instrumentation.http_url_connection.HttpUrlConnectionDecorator import spock.lang.Ignore import spock.lang.Requires +import spock.lang.Timeout import sun.net.www.protocol.https.HttpsURLConnectionImpl import static datadog.trace.agent.test.utils.ConfigUtils.withConfigOverride @@ -12,6 +13,7 @@ import static datadog.trace.agent.test.utils.TraceUtils.runUnderTrace import static datadog.trace.bootstrap.instrumentation.api.AgentTracer.activeScope import static datadog.trace.instrumentation.http_url_connection.HttpUrlConnectionInstrumentation.HttpUrlState.OPERATION_NAME +@Timeout(5) class HttpUrlConnectionTest extends HttpClientTest { static final RESPONSE = "Hello." @@ -25,6 +27,8 @@ class HttpUrlConnectionTest extends HttpClientTest { headers.each { connection.setRequestProperty(it.key, it.value) } connection.setRequestProperty("Connection", "close") connection.useCaches = true + connection.connectTimeout = 2000 + connection.readTimeout = 2000 def parentSpan = activeScope() def stream = connection.inputStream assert activeScope() == parentSpan diff --git a/dd-java-agent/instrumentation/http-url-connection/src/test/groovy/HttpUrlConnectionUseCachesFalseTest.groovy b/dd-java-agent/instrumentation/http-url-connection/src/test/groovy/HttpUrlConnectionUseCachesFalseTest.groovy index 36a9990946..eecc4660f2 100644 --- a/dd-java-agent/instrumentation/http-url-connection/src/test/groovy/HttpUrlConnectionUseCachesFalseTest.groovy +++ b/dd-java-agent/instrumentation/http-url-connection/src/test/groovy/HttpUrlConnectionUseCachesFalseTest.groovy @@ -1,8 +1,10 @@ import datadog.trace.agent.test.base.HttpClientTest import datadog.trace.instrumentation.http_url_connection.HttpUrlConnectionDecorator +import spock.lang.Timeout import static datadog.trace.bootstrap.instrumentation.api.AgentTracer.activeScope +@Timeout(5) class HttpUrlConnectionUseCachesFalseTest extends HttpClientTest { @Override @@ -13,6 +15,8 @@ class HttpUrlConnectionUseCachesFalseTest extends HttpClientTest { headers.each { connection.setRequestProperty(it.key, it.value) } connection.setRequestProperty("Connection", "close") connection.useCaches = false + connection.connectTimeout = 2000 + connection.readTimeout = 2000 def parentSpan = activeScope() def stream = connection.inputStream assert activeScope() == parentSpan diff --git a/dd-java-agent/instrumentation/http-url-connection/src/test/groovy/SpringRestTemplateTest.groovy b/dd-java-agent/instrumentation/http-url-connection/src/test/groovy/SpringRestTemplateTest.groovy index 97f208b809..1857c6d025 100644 --- a/dd-java-agent/instrumentation/http-url-connection/src/test/groovy/SpringRestTemplateTest.groovy +++ b/dd-java-agent/instrumentation/http-url-connection/src/test/groovy/SpringRestTemplateTest.groovy @@ -4,13 +4,24 @@ import org.springframework.http.HttpEntity import org.springframework.http.HttpHeaders import org.springframework.http.HttpMethod import org.springframework.http.ResponseEntity +import org.springframework.http.client.ClientHttpRequestFactory +import org.springframework.http.client.SimpleClientHttpRequestFactory import org.springframework.web.client.RestTemplate import spock.lang.Shared +import spock.lang.Timeout +@Timeout(5) class SpringRestTemplateTest extends HttpClientTest { @Shared - RestTemplate restTemplate = new RestTemplate() + ClientHttpRequestFactory factory = new SimpleClientHttpRequestFactory() + @Shared + RestTemplate restTemplate = new RestTemplate(factory) + + def setupSpec() { + factory.connectTimeout = 2000 + factory.readTimeout = 2000 + } @Override int doRequest(String method, URI uri, Map headers, Closure callback) { @@ -36,4 +47,10 @@ class SpringRestTemplateTest extends HttpClientTest { boolean testConnectionFailure() { false } + + @Override + boolean testRemoteConnection() { + // FIXME: exception wrapped in ResourceAccessException + return false + } } diff --git a/dd-java-agent/instrumentation/jax-rs-client-1.1/src/test/groovy/JaxRsClientV1Test.groovy b/dd-java-agent/instrumentation/jax-rs-client-1.1/src/test/groovy/JaxRsClientV1Test.groovy index 5cc989235e..c917f3b2d5 100644 --- a/dd-java-agent/instrumentation/jax-rs-client-1.1/src/test/groovy/JaxRsClientV1Test.groovy +++ b/dd-java-agent/instrumentation/jax-rs-client-1.1/src/test/groovy/JaxRsClientV1Test.groovy @@ -5,7 +5,9 @@ import com.sun.jersey.api.client.filter.LoggingFilter import datadog.trace.agent.test.base.HttpClientTest import datadog.trace.instrumentation.jaxrs.v1.JaxRsClientV1Decorator import spock.lang.Shared +import spock.lang.Timeout +@Timeout(5) class JaxRsClientV1Test extends HttpClientTest { @Shared diff --git a/dd-java-agent/instrumentation/jax-rs-client-2.0/jax-rs-client-2.0.gradle b/dd-java-agent/instrumentation/jax-rs-client-2.0/jax-rs-client-2.0.gradle index 6993d488a5..5cda555fc8 100644 --- a/dd-java-agent/instrumentation/jax-rs-client-2.0/jax-rs-client-2.0.gradle +++ b/dd-java-agent/instrumentation/jax-rs-client-2.0/jax-rs-client-2.0.gradle @@ -33,7 +33,8 @@ dependencies { testCompile group: 'javax.ws.rs', name: 'javax.ws.rs-api', version: '2.0.1' testCompile group: 'org.glassfish.jersey.core', name: 'jersey-client', version: '2.0' - testCompile group: 'org.jboss.resteasy', name: 'resteasy-client', version: '3.0.0.Final' + testCompile group: 'org.jboss.resteasy', name: 'resteasy-client', version: '3.0.5.Final' + // ^ This version has timeouts https://issues.redhat.com/browse/RESTEASY-975 testCompile group: 'org.apache.cxf', name: 'cxf-rt-rs-client', version: '3.1.0' // Doesn't work with CXF 3.0.x because their context is wrong: // https://github.com/apache/cxf/commit/335c7bad2436f08d6d54180212df5a52157c9f21 diff --git a/dd-java-agent/instrumentation/jax-rs-client-2.0/src/test/groovy/JaxRsClientAsyncTest.groovy b/dd-java-agent/instrumentation/jax-rs-client-2.0/src/test/groovy/JaxRsClientAsyncTest.groovy index 1a9cf50508..65fcdfd5bc 100644 --- a/dd-java-agent/instrumentation/jax-rs-client-2.0/src/test/groovy/JaxRsClientAsyncTest.groovy +++ b/dd-java-agent/instrumentation/jax-rs-client-2.0/src/test/groovy/JaxRsClientAsyncTest.groovy @@ -1,8 +1,11 @@ import datadog.trace.agent.test.base.HttpClientTest import datadog.trace.instrumentation.jaxrs.JaxRsClientDecorator import org.apache.cxf.jaxrs.client.spec.ClientBuilderImpl +import org.glassfish.jersey.client.ClientConfig +import org.glassfish.jersey.client.ClientProperties import org.glassfish.jersey.client.JerseyClientBuilder import org.jboss.resteasy.client.jaxrs.ResteasyClientBuilder +import spock.lang.Timeout import javax.ws.rs.client.AsyncInvoker import javax.ws.rs.client.Client @@ -12,6 +15,7 @@ import javax.ws.rs.client.InvocationCallback import javax.ws.rs.client.WebTarget import javax.ws.rs.core.MediaType import javax.ws.rs.core.Response +import java.util.concurrent.TimeUnit abstract class JaxRsClientAsyncTest extends HttpClientTest { @@ -51,11 +55,15 @@ abstract class JaxRsClientAsyncTest extends HttpClientTest { abstract ClientBuilder builder() } +@Timeout(5) class JerseyClientAsyncTest extends JaxRsClientAsyncTest { @Override ClientBuilder builder() { - return new JerseyClientBuilder() + ClientConfig config = new ClientConfig() + config.property(ClientProperties.CONNECT_TIMEOUT, 2000) + config.property(ClientProperties.READ_TIMEOUT, 2000) + return new JerseyClientBuilder().withConfig(config) } boolean testCircularRedirects() { @@ -63,11 +71,14 @@ class JerseyClientAsyncTest extends JaxRsClientAsyncTest { } } +@Timeout(5) class ResteasyClientAsyncTest extends JaxRsClientAsyncTest { @Override ClientBuilder builder() { return new ResteasyClientBuilder() + .establishConnectionTimeout(2, TimeUnit.SECONDS) + .socketTimeout(2, TimeUnit.SECONDS) } boolean testRedirects() { @@ -75,6 +86,7 @@ class ResteasyClientAsyncTest extends JaxRsClientAsyncTest { } } +@Timeout(5) class CxfClientAsyncTest extends JaxRsClientAsyncTest { @Override @@ -89,4 +101,9 @@ class CxfClientAsyncTest extends JaxRsClientAsyncTest { boolean testConnectionFailure() { false } + + boolean testRemoteConnection() { + // FIXME: span not reported correctly. + false + } } diff --git a/dd-java-agent/instrumentation/jax-rs-client-2.0/src/test/groovy/JaxRsClientTest.groovy b/dd-java-agent/instrumentation/jax-rs-client-2.0/src/test/groovy/JaxRsClientTest.groovy index 0de0e1c075..65372a5c1a 100644 --- a/dd-java-agent/instrumentation/jax-rs-client-2.0/src/test/groovy/JaxRsClientTest.groovy +++ b/dd-java-agent/instrumentation/jax-rs-client-2.0/src/test/groovy/JaxRsClientTest.groovy @@ -1,8 +1,11 @@ import datadog.trace.agent.test.base.HttpClientTest import datadog.trace.instrumentation.jaxrs.JaxRsClientDecorator import org.apache.cxf.jaxrs.client.spec.ClientBuilderImpl +import org.glassfish.jersey.client.ClientConfig +import org.glassfish.jersey.client.ClientProperties import org.glassfish.jersey.client.JerseyClientBuilder import org.jboss.resteasy.client.jaxrs.ResteasyClientBuilder +import spock.lang.Timeout import javax.ws.rs.client.Client import javax.ws.rs.client.ClientBuilder @@ -11,6 +14,7 @@ import javax.ws.rs.client.Invocation import javax.ws.rs.client.WebTarget import javax.ws.rs.core.MediaType import javax.ws.rs.core.Response +import java.util.concurrent.TimeUnit abstract class JaxRsClientTest extends HttpClientTest { @@ -41,11 +45,15 @@ abstract class JaxRsClientTest extends HttpClientTest { abstract ClientBuilder builder() } +@Timeout(5) class JerseyClientTest extends JaxRsClientTest { @Override ClientBuilder builder() { - return new JerseyClientBuilder() + ClientConfig config = new ClientConfig() + config.property(ClientProperties.CONNECT_TIMEOUT, 2000) + config.property(ClientProperties.READ_TIMEOUT, 2000) + return new JerseyClientBuilder().withConfig(config) } boolean testCircularRedirects() { @@ -53,24 +61,29 @@ class JerseyClientTest extends JaxRsClientTest { } } +@Timeout(5) class ResteasyClientTest extends JaxRsClientTest { @Override ClientBuilder builder() { return new ResteasyClientBuilder() + .establishConnectionTimeout(2, TimeUnit.SECONDS) + .socketTimeout(2, TimeUnit.SECONDS) } boolean testRedirects() { false } - } +@Timeout(5) class CxfClientTest extends JaxRsClientTest { @Override ClientBuilder builder() { return new ClientBuilderImpl() +// .property(ClientImpl.HTTP_CONNECTION_TIMEOUT_PROP, 2000L) +// .property(ClientImpl.HTTP_RECEIVE_TIMEOUT_PROP, 2000L) } boolean testRedirects() { @@ -80,4 +93,9 @@ class CxfClientTest extends JaxRsClientTest { boolean testConnectionFailure() { false } + + boolean testRemoteConnection() { + // FIXME: span not reported correctly. + false + } } diff --git a/dd-java-agent/instrumentation/netty-4.0/src/test/groovy/Netty40ClientTest.groovy b/dd-java-agent/instrumentation/netty-4.0/src/test/groovy/Netty40ClientTest.groovy index dcda69937e..095265371e 100644 --- a/dd-java-agent/instrumentation/netty-4.0/src/test/groovy/Netty40ClientTest.groovy +++ b/dd-java-agent/instrumentation/netty-4.0/src/test/groovy/Netty40ClientTest.groovy @@ -7,6 +7,7 @@ import org.asynchttpclient.DefaultAsyncHttpClientConfig import org.asynchttpclient.Response import spock.lang.AutoCleanup import spock.lang.Shared +import spock.lang.Timeout import java.util.concurrent.ExecutionException import java.util.concurrent.TimeUnit @@ -16,6 +17,7 @@ import static datadog.trace.agent.test.utils.TraceUtils.basicSpan import static datadog.trace.agent.test.utils.TraceUtils.runUnderTrace import static org.asynchttpclient.Dsl.asyncHttpClient +@Timeout(5) class Netty40ClientTest extends HttpClientTest { @Shared @@ -60,6 +62,11 @@ class Netty40ClientTest extends HttpClientTest { false } + @Override + boolean testRemoteConnection() { + return false + } + def "connection error (unopened port)"() { given: def uri = new URI("http://localhost:$UNUSABLE_PORT/") diff --git a/dd-java-agent/instrumentation/netty-4.1/src/test/groovy/Netty41ClientTest.groovy b/dd-java-agent/instrumentation/netty-4.1/src/test/groovy/Netty41ClientTest.groovy index 9f3d2d14be..641b038243 100644 --- a/dd-java-agent/instrumentation/netty-4.1/src/test/groovy/Netty41ClientTest.groovy +++ b/dd-java-agent/instrumentation/netty-4.1/src/test/groovy/Netty41ClientTest.groovy @@ -17,6 +17,7 @@ import org.asynchttpclient.DefaultAsyncHttpClientConfig import org.asynchttpclient.Response import spock.lang.Retry import spock.lang.Shared +import spock.lang.Timeout import java.util.concurrent.ExecutionException import java.util.concurrent.TimeUnit @@ -27,6 +28,7 @@ import static datadog.trace.agent.test.utils.TraceUtils.runUnderTrace import static org.asynchttpclient.Dsl.asyncHttpClient @Retry +@Timeout(5) class Netty41ClientTest extends HttpClientTest { @Shared @@ -70,6 +72,11 @@ class Netty41ClientTest extends HttpClientTest { false } + @Override + boolean testRemoteConnection() { + return false + } + def "connection error (unopened port)"() { given: def uri = new URI("http://localhost:$UNUSABLE_PORT/") diff --git a/dd-java-agent/instrumentation/okhttp-3/src/test/groovy/OkHttp3Test.groovy b/dd-java-agent/instrumentation/okhttp-3/src/test/groovy/OkHttp3Test.groovy index a344c2fc98..7ff99a0b51 100644 --- a/dd-java-agent/instrumentation/okhttp-3/src/test/groovy/OkHttp3Test.groovy +++ b/dd-java-agent/instrumentation/okhttp-3/src/test/groovy/OkHttp3Test.groovy @@ -6,10 +6,18 @@ import okhttp3.OkHttpClient import okhttp3.Request import okhttp3.RequestBody import okhttp3.internal.http.HttpMethod +import spock.lang.Timeout +import java.util.concurrent.TimeUnit + +@Timeout(5) class OkHttp3Test extends HttpClientTest { - def client = new OkHttpClient() + def client = new OkHttpClient.Builder() + .connectTimeout(2, TimeUnit.SECONDS) + .readTimeout(2, TimeUnit.SECONDS) + .writeTimeout(2, TimeUnit.SECONDS) + .build() @Override int doRequest(String method, URI uri, Map headers, Closure callback) { diff --git a/dd-java-agent/instrumentation/play-2.4/src/test/groovy/client/PlayWSClientTest.groovy b/dd-java-agent/instrumentation/play-2.4/src/test/groovy/client/PlayWSClientTest.groovy index 343e3b60b2..7368363d60 100644 --- a/dd-java-agent/instrumentation/play-2.4/src/test/groovy/client/PlayWSClientTest.groovy +++ b/dd-java-agent/instrumentation/play-2.4/src/test/groovy/client/PlayWSClientTest.groovy @@ -6,9 +6,11 @@ import play.libs.ws.WS import spock.lang.AutoCleanup import spock.lang.Shared import spock.lang.Subject +import spock.lang.Timeout // Play 2.6+ uses a separately versioned client that shades the underlying dependency // This means our built in instrumentation won't work. +@Timeout(5) class PlayWSClientTest extends HttpClientTest { @Subject @Shared @@ -50,4 +52,9 @@ class PlayWSClientTest extends HttpClientTest { boolean testConnectionFailure() { false } + + @Override + boolean testRemoteConnection() { + return false + } } diff --git a/dd-java-agent/instrumentation/play-ws-1/src/test/groovy/PlayWSClientTest.groovy b/dd-java-agent/instrumentation/play-ws-1/src/test/groovy/PlayWSClientTest.groovy index 97b226c20d..889d108df3 100644 --- a/dd-java-agent/instrumentation/play-ws-1/src/test/groovy/PlayWSClientTest.groovy +++ b/dd-java-agent/instrumentation/play-ws-1/src/test/groovy/PlayWSClientTest.groovy @@ -8,9 +8,11 @@ import scala.concurrent.ExecutionContext import scala.concurrent.Future import scala.concurrent.duration.Duration import spock.lang.Shared +import spock.lang.Timeout import java.util.concurrent.TimeUnit +@Timeout(5) class PlayJavaWSClientTest extends PlayWSClientTestBase { @Shared StandaloneWSClient wsClient @@ -37,6 +39,7 @@ class PlayJavaWSClientTest extends PlayWSClientTestBase { } } +@Timeout(5) class PlayJavaStreamedWSClientTest extends PlayWSClientTestBase { @Shared StandaloneWSClient wsClient @@ -66,6 +69,7 @@ class PlayJavaStreamedWSClientTest extends PlayWSClientTestBase { } } +@Timeout(5) class PlayScalaWSClientTest extends PlayWSClientTestBase { @Shared play.api.libs.ws.StandaloneWSClient wsClient @@ -96,6 +100,7 @@ class PlayScalaWSClientTest extends PlayWSClientTestBase { } } +@Timeout(5) class PlayScalaStreamedWSClientTest extends PlayWSClientTestBase { @Shared play.api.libs.ws.StandaloneWSClient wsClient diff --git a/dd-java-agent/instrumentation/ratpack-1.4/src/test/groovy/client/RatpackHttpClientTest.groovy b/dd-java-agent/instrumentation/ratpack-1.4/src/test/groovy/client/RatpackHttpClientTest.groovy index f4308659f3..11c8613ee7 100644 --- a/dd-java-agent/instrumentation/ratpack-1.4/src/test/groovy/client/RatpackHttpClientTest.groovy +++ b/dd-java-agent/instrumentation/ratpack-1.4/src/test/groovy/client/RatpackHttpClientTest.groovy @@ -7,7 +7,11 @@ import ratpack.http.client.HttpClient import ratpack.test.exec.ExecHarness import spock.lang.AutoCleanup import spock.lang.Shared +import spock.lang.Timeout +import java.time.Duration + +@Timeout(5) class RatpackHttpClientTest extends HttpClientTest { @AutoCleanup @@ -15,12 +19,16 @@ class RatpackHttpClientTest extends HttpClientTest { ExecHarness exec = ExecHarness.harness() @Shared - def client = HttpClient.of {} + def client = HttpClient.of { + it.readTimeout(Duration.ofSeconds(2)) + // Connect timeout added in 1.5 + } @Override int doRequest(String method, URI uri, Map headers, Closure callback) { ExecResult result = exec.yield { def resp = client.request(uri) { spec -> + spec.connectTimeout(Duration.ofSeconds(2)) spec.method(method) spec.headers { headersSpec -> headers.entrySet().each { @@ -55,4 +63,9 @@ class RatpackHttpClientTest extends HttpClientTest { boolean testConnectionFailure() { false } + + @Override + boolean testRemoteConnection() { + return false + } } diff --git a/dd-java-agent/instrumentation/spring-webflux-5/src/test/groovy/dd/trace/instrumentation/springwebflux/client/SpringWebfluxHttpClientTest.groovy b/dd-java-agent/instrumentation/spring-webflux-5/src/test/groovy/dd/trace/instrumentation/springwebflux/client/SpringWebfluxHttpClientTest.groovy index 7e1b4b266c..57e8bf8abd 100644 --- a/dd-java-agent/instrumentation/spring-webflux-5/src/test/groovy/dd/trace/instrumentation/springwebflux/client/SpringWebfluxHttpClientTest.groovy +++ b/dd-java-agent/instrumentation/spring-webflux-5/src/test/groovy/dd/trace/instrumentation/springwebflux/client/SpringWebfluxHttpClientTest.groovy @@ -11,9 +11,11 @@ import org.springframework.http.HttpMethod import org.springframework.web.reactive.function.client.ClientResponse import org.springframework.web.reactive.function.client.WebClient import spock.lang.Shared +import spock.lang.Timeout import static datadog.trace.bootstrap.instrumentation.api.AgentTracer.activeSpan +@Timeout(5) class SpringWebfluxHttpClientTest extends HttpClientTest { @Shared @@ -92,4 +94,9 @@ class SpringWebfluxHttpClientTest extends HttpClientTest { boolean testConnectionFailure() { false } + + boolean testRemoteConnection() { + // FIXME: figure out how to configure timeouts. + false + } } diff --git a/dd-java-agent/instrumentation/vertx/src/test/groovy/client/VertxHttpClientTest.groovy b/dd-java-agent/instrumentation/vertx/src/test/groovy/client/VertxHttpClientTest.groovy index f1012654ed..738843db9b 100644 --- a/dd-java-agent/instrumentation/vertx/src/test/groovy/client/VertxHttpClientTest.groovy +++ b/dd-java-agent/instrumentation/vertx/src/test/groovy/client/VertxHttpClientTest.groovy @@ -4,7 +4,7 @@ import datadog.trace.agent.test.base.HttpClientTest import datadog.trace.instrumentation.netty41.client.NettyHttpClientDecorator import io.vertx.core.Vertx import io.vertx.core.VertxOptions -import io.vertx.core.http.HttpClient +import io.vertx.core.http.HttpClientOptions import io.vertx.core.http.HttpClientResponse import io.vertx.core.http.HttpMethod import spock.lang.Shared @@ -16,9 +16,11 @@ import java.util.concurrent.CompletableFuture class VertxHttpClientTest extends HttpClientTest { @Shared - Vertx vertx = Vertx.vertx(new VertxOptions()) + def vertx = Vertx.vertx(new VertxOptions()) @Shared - HttpClient httpClient = vertx.createHttpClient() + def clientOptions = new HttpClientOptions().setConnectTimeout(2000).setIdleTimeout(2000) + @Shared + def httpClient = vertx.createHttpClient(clientOptions) @Override int doRequest(String method, URI uri, Map headers, Closure callback) { @@ -53,4 +55,9 @@ class VertxHttpClientTest extends HttpClientTest { boolean testConnectionFailure() { false } + + boolean testRemoteConnection() { + // FIXME: figure out how to configure timeouts. + false + } } diff --git a/dd-java-agent/instrumentation/vertx/src/test/groovy/client/VertxRxCircuitBreakerWebClientTest.groovy b/dd-java-agent/instrumentation/vertx/src/test/groovy/client/VertxRxCircuitBreakerWebClientTest.groovy index 73d6924aed..6b7fc721f6 100644 --- a/dd-java-agent/instrumentation/vertx/src/test/groovy/client/VertxRxCircuitBreakerWebClientTest.groovy +++ b/dd-java-agent/instrumentation/vertx/src/test/groovy/client/VertxRxCircuitBreakerWebClientTest.groovy @@ -5,6 +5,7 @@ import datadog.trace.instrumentation.netty41.client.NettyHttpClientDecorator import io.vertx.circuitbreaker.CircuitBreakerOptions import io.vertx.core.VertxOptions import io.vertx.core.http.HttpMethod +import io.vertx.ext.web.client.WebClientOptions import io.vertx.reactivex.circuitbreaker.CircuitBreaker import io.vertx.reactivex.core.Vertx import io.vertx.reactivex.ext.web.client.WebClient @@ -19,7 +20,9 @@ class VertxRxCircuitBreakerWebClientTest extends HttpClientTest { @Shared Vertx vertx = Vertx.vertx(new VertxOptions()) @Shared - WebClient client = WebClient.create(vertx) + def clientOptions = new WebClientOptions().setConnectTimeout(2000).setIdleTimeout(2000) + @Shared + WebClient client = WebClient.create(vertx, clientOptions) @Shared CircuitBreaker breaker = CircuitBreaker.create("my-circuit-breaker", vertx, new CircuitBreakerOptions() @@ -69,4 +72,9 @@ class VertxRxCircuitBreakerWebClientTest extends HttpClientTest { boolean testConnectionFailure() { false } + + boolean testRemoteConnection() { + // FIXME: figure out how to configure timeouts. + false + } } diff --git a/dd-java-agent/instrumentation/vertx/src/test/groovy/client/VertxRxWebClientTest.groovy b/dd-java-agent/instrumentation/vertx/src/test/groovy/client/VertxRxWebClientTest.groovy index a5f8054f2d..acde3648f7 100644 --- a/dd-java-agent/instrumentation/vertx/src/test/groovy/client/VertxRxWebClientTest.groovy +++ b/dd-java-agent/instrumentation/vertx/src/test/groovy/client/VertxRxWebClientTest.groovy @@ -4,6 +4,7 @@ import datadog.trace.agent.test.base.HttpClientTest import datadog.trace.instrumentation.netty41.client.NettyHttpClientDecorator import io.vertx.core.VertxOptions import io.vertx.core.http.HttpMethod +import io.vertx.ext.web.client.WebClientOptions import io.vertx.reactivex.core.Vertx import io.vertx.reactivex.ext.web.client.WebClient import spock.lang.Shared @@ -15,7 +16,9 @@ class VertxRxWebClientTest extends HttpClientTest { @Shared Vertx vertx = Vertx.vertx(new VertxOptions()) @Shared - WebClient client = WebClient.create(vertx) + def clientOptions = new WebClientOptions().setConnectTimeout(2000).setIdleTimeout(2000) + @Shared + WebClient client = WebClient.create(vertx, clientOptions) @Override int doRequest(String method, URI uri, Map headers, Closure callback) { @@ -48,4 +51,9 @@ class VertxRxWebClientTest extends HttpClientTest { boolean testConnectionFailure() { false } + + boolean testRemoteConnection() { + // FIXME: figure out how to configure timeouts. + false + } } diff --git a/dd-java-agent/testing/src/main/groovy/datadog/trace/agent/test/base/HttpClientTest.groovy b/dd-java-agent/testing/src/main/groovy/datadog/trace/agent/test/base/HttpClientTest.groovy index 850c767628..9132158770 100644 --- a/dd-java-agent/testing/src/main/groovy/datadog/trace/agent/test/base/HttpClientTest.groovy +++ b/dd-java-agent/testing/src/main/groovy/datadog/trace/agent/test/base/HttpClientTest.groovy @@ -109,7 +109,7 @@ abstract class HttpClientTest extends AgentTestRunner { server.distributedRequestTrace(it, 0, trace(1).last()) trace(1, size(2)) { basicSpan(it, 0, "parent") - clientSpan(it, 1, span(0), method, false) + clientSpan(it, 1, span(0), method) } } @@ -179,7 +179,7 @@ abstract class HttpClientTest extends AgentTestRunner { trace(0, size(3)) { basicSpan(it, 0, "parent") basicSpan(it, 1, "child", span(0)) - clientSpan(it, 2, span(0), method, false) + clientSpan(it, 2, span(0), method) } } @@ -201,7 +201,7 @@ abstract class HttpClientTest extends AgentTestRunner { // only one trace (client). assertTraces(2) { trace(0, size(1)) { - clientSpan(it, 0, null, method, false) + clientSpan(it, 0, null, method) } trace(1, 1) { basicSpan(it, 0, "callback") @@ -262,8 +262,7 @@ abstract class HttpClientTest extends AgentTestRunner { def "basic #method request with circular redirects"() { given: - assumeTrue(testRedirects()) - assumeTrue(testCircularRedirects()) + assumeTrue(testRedirects() && testCircularRedirects()) def uri = server.address.resolve("/circular-redirect") when: @@ -312,6 +311,75 @@ abstract class HttpClientTest extends AgentTestRunner { method = "GET" } + def "connection error dropped request"() { + given: + assumeTrue(testRemoteConnection()) + // https://stackoverflow.com/a/100859 + def uri = new URI("http://www.google.com:81/") + + when: + runUnderTrace("parent") { + doRequest(method, uri) + } + + then: + def ex = thrown(Exception) + def thrownException = ex instanceof ExecutionException ? ex.cause : ex + assertTraces(1) { + trace(0, size(2)) { + basicSpan(it, 0, "parent", null, thrownException) + clientSpan(it, 1, span(0), method, false, false, uri, null, thrownException) + } + } + + where: + method = "HEAD" + } + + def "connection error non routable address"() { + given: + assumeTrue(testRemoteConnection()) + def uri = new URI("https://192.0.2.1/") + + when: + runUnderTrace("parent") { + doRequest(method, uri) + } + + then: + def ex = thrown(Exception) + def thrownException = ex instanceof ExecutionException ? ex.cause : ex + assertTraces(1) { + trace(0, size(2)) { + basicSpan(it, 0, "parent", null, thrownException) + clientSpan(it, 1, span(0), method, false, false, uri, null, thrownException) + } + } + + where: + method = "HEAD" + } + + def "test https request"() { + given: + assumeTrue(testRemoteConnection()) + def uri = new URI("https://www.google.com/") + + when: + def status = doRequest(method, uri) + + then: + status == 200 + assertTraces(1) { + trace(0, size(1)) { + clientSpan(it, 0, null, method, false, false, uri) + } + } + + where: + method = "HEAD" + } + // parent span must be cast otherwise it breaks debugging classloading (junit loads it early) void clientSpan(TraceAssert trace, int index, Object parentSpan, String method = "GET", boolean renameService = false, boolean tagQueryString = false, URI uri = server.address.resolve("/success"), Integer status = 200, Throwable exception = null) { trace.span(index) { @@ -320,7 +388,7 @@ abstract class HttpClientTest extends AgentTestRunner { } else { childOf((DDSpan) parentSpan) } - serviceName renameService ? "localhost" : "unnamed-java-app" + serviceName renameService ? uri.host : "unnamed-java-app" operationName expectedOperationName() resourceName "$method $uri.path" spanType DDSpanTypes.HTTP_CLIENT @@ -328,9 +396,9 @@ abstract class HttpClientTest extends AgentTestRunner { tags { "$Tags.COMPONENT" component "$Tags.SPAN_KIND" Tags.SPAN_KIND_CLIENT - "$Tags.PEER_HOSTNAME" "localhost" + "$Tags.PEER_HOSTNAME" uri.host "$Tags.PEER_HOST_IPV4" { it == null || it == "127.0.0.1" } // Optional - "$Tags.PEER_PORT" uri.port + "$Tags.PEER_PORT" uri.port > 0 ? uri.port : { it == null || it == 443 } // Optional "$Tags.HTTP_URL" "${uri.resolve(uri.path)}" "$Tags.HTTP_METHOD" method if (status) { @@ -368,6 +436,10 @@ abstract class HttpClientTest extends AgentTestRunner { true } + boolean testRemoteConnection() { + true + } + boolean testCallbackWithParent() { // FIXME: this hack is here because callback with parent is broken in play-ws when the stream() // function is used. There is no way to stop a test from a derived class hence the flag diff --git a/dd-java-agent/testing/src/main/groovy/datadog/trace/agent/test/server/http/TestHttpServer.groovy b/dd-java-agent/testing/src/main/groovy/datadog/trace/agent/test/server/http/TestHttpServer.groovy index 8acff9151b..1b9ecd64aa 100644 --- a/dd-java-agent/testing/src/main/groovy/datadog/trace/agent/test/server/http/TestHttpServer.groovy +++ b/dd-java-agent/testing/src/main/groovy/datadog/trace/agent/test/server/http/TestHttpServer.groovy @@ -22,17 +22,14 @@ import static datadog.trace.bootstrap.instrumentation.api.AgentTracer.startSpan class TestHttpServer implements AutoCloseable { - static TestHttpServer httpServer(boolean start = true, - @DelegatesTo(value = TestHttpServer, strategy = Closure.DELEGATE_FIRST) Closure spec) { + static TestHttpServer httpServer(@DelegatesTo(value = TestHttpServer, strategy = Closure.DELEGATE_FIRST) Closure spec) { def server = new TestHttpServer() def clone = (Closure) spec.clone() clone.delegate = server clone.resolveStrategy = Closure.DELEGATE_FIRST clone(server) - if (start) { - server.start() - } + server.start() return server } diff --git a/dd-java-agent/testing/src/test/groovy/server/ServerTest.groovy b/dd-java-agent/testing/src/test/groovy/server/ServerTest.groovy index 0476295a99..3c2109f329 100644 --- a/dd-java-agent/testing/src/test/groovy/server/ServerTest.groovy +++ b/dd-java-agent/testing/src/test/groovy/server/ServerTest.groovy @@ -18,12 +18,12 @@ class ServerTest extends AgentTestRunner { def "test server lifecycle"() { setup: - def server = httpServer(startAutomatically) { + def server = httpServer { handlers {} } expect: - server.internalServer.isRunning() == startAutomatically + server.internalServer.isRunning() when: server.start() @@ -51,9 +51,6 @@ class ServerTest extends AgentTestRunner { cleanup: server.stop() - - where: - startAutomatically << [true, false] } def "server 404's with no handlers"() { From d86cb51aaad19431c537bfcd7d0de91183ce47dd Mon Sep 17 00:00:00 2001 From: Tyler Benson Date: Mon, 16 Mar 2020 09:10:06 -0700 Subject: [PATCH 02/43] Fix more tests and extract timeout values to constant --- .../ApacheHttpClientResponseHandlerTest.groovy | 4 ++-- .../src/test/groovy/ApacheHttpClientTest.groovy | 4 ++-- .../src/test/groovy/CommonsHttpClientTest.groovy | 4 ++-- .../test/groovy/AbstractGoogleHttpClientTest.groovy | 4 ++-- .../HttpUrlConnectionResponseCodeOnlyTest.groovy | 4 ++-- .../src/test/groovy/HttpUrlConnectionTest.groovy | 4 ++-- .../HttpUrlConnectionUseCachesFalseTest.groovy | 4 ++-- .../src/test/groovy/SpringRestTemplateTest.groovy | 4 ++-- .../src/test/groovy/JaxRsClientV1Test.groovy | 2 ++ .../src/test/groovy/JaxRsClientAsyncTest.groovy | 8 ++++---- .../src/test/groovy/JaxRsClientTest.groovy | 12 ++++++------ .../okhttp-3/src/test/groovy/OkHttp3Test.groovy | 6 +++--- .../test/groovy/client/VertxHttpClientTest.groovy | 2 +- .../client/VertxRxCircuitBreakerWebClientTest.groovy | 2 +- .../test/groovy/client/VertxRxWebClientTest.groovy | 2 +- .../trace/agent/test/base/HttpClientTest.groovy | 2 ++ 16 files changed, 36 insertions(+), 32 deletions(-) diff --git a/dd-java-agent/instrumentation/apache-httpclient-4/src/test/groovy/ApacheHttpClientResponseHandlerTest.groovy b/dd-java-agent/instrumentation/apache-httpclient-4/src/test/groovy/ApacheHttpClientResponseHandlerTest.groovy index b7a03b697c..37bfdbf6a9 100644 --- a/dd-java-agent/instrumentation/apache-httpclient-4/src/test/groovy/ApacheHttpClientResponseHandlerTest.groovy +++ b/dd-java-agent/instrumentation/apache-httpclient-4/src/test/groovy/ApacheHttpClientResponseHandlerTest.groovy @@ -25,8 +25,8 @@ class ApacheHttpClientResponseHandlerTest extends HttpClientTest { def setupSpec() { HttpParams httpParams = client.getParams() - HttpConnectionParams.setConnectionTimeout(httpParams, 2000) - HttpConnectionParams.setSoTimeout(httpParams, 2000) + HttpConnectionParams.setConnectionTimeout(httpParams, CONNECT_TIMEOUT_MS) + HttpConnectionParams.setSoTimeout(httpParams, READ_TIMEOUT_MS) } @Override diff --git a/dd-java-agent/instrumentation/apache-httpclient-4/src/test/groovy/ApacheHttpClientTest.groovy b/dd-java-agent/instrumentation/apache-httpclient-4/src/test/groovy/ApacheHttpClientTest.groovy index 49841e1c10..71023ef408 100644 --- a/dd-java-agent/instrumentation/apache-httpclient-4/src/test/groovy/ApacheHttpClientTest.groovy +++ b/dd-java-agent/instrumentation/apache-httpclient-4/src/test/groovy/ApacheHttpClientTest.groovy @@ -18,8 +18,8 @@ abstract class ApacheHttpClientTest extends HttpClientTes def setupSpec() { HttpParams httpParams = client.getParams() - HttpConnectionParams.setConnectionTimeout(httpParams, 2000) - HttpConnectionParams.setSoTimeout(httpParams, 2000) + HttpConnectionParams.setConnectionTimeout(httpParams, CONNECT_TIMEOUT_MS) + HttpConnectionParams.setSoTimeout(httpParams, READ_TIMEOUT_MS) } @Override diff --git a/dd-java-agent/instrumentation/commons-httpclient-2/src/test/groovy/CommonsHttpClientTest.groovy b/dd-java-agent/instrumentation/commons-httpclient-2/src/test/groovy/CommonsHttpClientTest.groovy index dc1456d812..abcbfda361 100644 --- a/dd-java-agent/instrumentation/commons-httpclient-2/src/test/groovy/CommonsHttpClientTest.groovy +++ b/dd-java-agent/instrumentation/commons-httpclient-2/src/test/groovy/CommonsHttpClientTest.groovy @@ -18,8 +18,8 @@ class CommonsHttpClientTest extends HttpClientTest { HttpClient client = new HttpClient() def setupSpec() { - client.setConnectionTimeout(2000) - client.setTimeout(2000) + client.setConnectionTimeout(CONNECT_TIMEOUT_MS) + client.setTimeout(READ_TIMEOUT_MS) } @Override diff --git a/dd-java-agent/instrumentation/google-http-client/src/test/groovy/AbstractGoogleHttpClientTest.groovy b/dd-java-agent/instrumentation/google-http-client/src/test/groovy/AbstractGoogleHttpClientTest.groovy index 0ffa54e67b..7faf438639 100644 --- a/dd-java-agent/instrumentation/google-http-client/src/test/groovy/AbstractGoogleHttpClientTest.groovy +++ b/dd-java-agent/instrumentation/google-http-client/src/test/groovy/AbstractGoogleHttpClientTest.groovy @@ -23,8 +23,8 @@ abstract class AbstractGoogleHttpClientTest extends HttpClientTest { GenericUrl genericUrl = new GenericUrl(uri) HttpRequest request = requestFactory.buildRequest(method, genericUrl, null) - request.connectTimeout = 2000 - request.readTimeout = 2000 + request.connectTimeout = CONNECT_TIMEOUT_MS + request.readTimeout = READ_TIMEOUT_MS request.getHeaders().putAll(headers) request.setThrowExceptionOnExecuteError(throwExceptionOnError) diff --git a/dd-java-agent/instrumentation/http-url-connection/src/test/groovy/HttpUrlConnectionResponseCodeOnlyTest.groovy b/dd-java-agent/instrumentation/http-url-connection/src/test/groovy/HttpUrlConnectionResponseCodeOnlyTest.groovy index 0b8d08dac0..574709f35a 100644 --- a/dd-java-agent/instrumentation/http-url-connection/src/test/groovy/HttpUrlConnectionResponseCodeOnlyTest.groovy +++ b/dd-java-agent/instrumentation/http-url-connection/src/test/groovy/HttpUrlConnectionResponseCodeOnlyTest.groovy @@ -10,8 +10,8 @@ class HttpUrlConnectionResponseCodeOnlyTest extends HttpClientTest { HttpURLConnection connection = uri.toURL().openConnection() try { connection.setRequestMethod(method) - connection.connectTimeout = 2000 - connection.readTimeout = 2000 + connection.connectTimeout = CONNECT_TIMEOUT_MS + connection.readTimeout = READ_TIMEOUT_MS headers.each { connection.setRequestProperty(it.key, it.value) } connection.setRequestProperty("Connection", "close") return connection.getResponseCode() diff --git a/dd-java-agent/instrumentation/http-url-connection/src/test/groovy/HttpUrlConnectionTest.groovy b/dd-java-agent/instrumentation/http-url-connection/src/test/groovy/HttpUrlConnectionTest.groovy index 0da17b75d1..05924531f5 100644 --- a/dd-java-agent/instrumentation/http-url-connection/src/test/groovy/HttpUrlConnectionTest.groovy +++ b/dd-java-agent/instrumentation/http-url-connection/src/test/groovy/HttpUrlConnectionTest.groovy @@ -27,8 +27,8 @@ class HttpUrlConnectionTest extends HttpClientTest { headers.each { connection.setRequestProperty(it.key, it.value) } connection.setRequestProperty("Connection", "close") connection.useCaches = true - connection.connectTimeout = 2000 - connection.readTimeout = 2000 + connection.connectTimeout = CONNECT_TIMEOUT_MS + connection.readTimeout = READ_TIMEOUT_MS def parentSpan = activeScope() def stream = connection.inputStream assert activeScope() == parentSpan diff --git a/dd-java-agent/instrumentation/http-url-connection/src/test/groovy/HttpUrlConnectionUseCachesFalseTest.groovy b/dd-java-agent/instrumentation/http-url-connection/src/test/groovy/HttpUrlConnectionUseCachesFalseTest.groovy index eecc4660f2..58d851765d 100644 --- a/dd-java-agent/instrumentation/http-url-connection/src/test/groovy/HttpUrlConnectionUseCachesFalseTest.groovy +++ b/dd-java-agent/instrumentation/http-url-connection/src/test/groovy/HttpUrlConnectionUseCachesFalseTest.groovy @@ -15,8 +15,8 @@ class HttpUrlConnectionUseCachesFalseTest extends HttpClientTest { headers.each { connection.setRequestProperty(it.key, it.value) } connection.setRequestProperty("Connection", "close") connection.useCaches = false - connection.connectTimeout = 2000 - connection.readTimeout = 2000 + connection.connectTimeout = CONNECT_TIMEOUT_MS + connection.readTimeout = READ_TIMEOUT_MS def parentSpan = activeScope() def stream = connection.inputStream assert activeScope() == parentSpan diff --git a/dd-java-agent/instrumentation/http-url-connection/src/test/groovy/SpringRestTemplateTest.groovy b/dd-java-agent/instrumentation/http-url-connection/src/test/groovy/SpringRestTemplateTest.groovy index 1857c6d025..765b052119 100644 --- a/dd-java-agent/instrumentation/http-url-connection/src/test/groovy/SpringRestTemplateTest.groovy +++ b/dd-java-agent/instrumentation/http-url-connection/src/test/groovy/SpringRestTemplateTest.groovy @@ -19,8 +19,8 @@ class SpringRestTemplateTest extends HttpClientTest { RestTemplate restTemplate = new RestTemplate(factory) def setupSpec() { - factory.connectTimeout = 2000 - factory.readTimeout = 2000 + factory.connectTimeout = CONNECT_TIMEOUT_MS + factory.readTimeout = READ_TIMEOUT_MS } @Override diff --git a/dd-java-agent/instrumentation/jax-rs-client-1.1/src/test/groovy/JaxRsClientV1Test.groovy b/dd-java-agent/instrumentation/jax-rs-client-1.1/src/test/groovy/JaxRsClientV1Test.groovy index c917f3b2d5..d6fceda611 100644 --- a/dd-java-agent/instrumentation/jax-rs-client-1.1/src/test/groovy/JaxRsClientV1Test.groovy +++ b/dd-java-agent/instrumentation/jax-rs-client-1.1/src/test/groovy/JaxRsClientV1Test.groovy @@ -14,6 +14,8 @@ class JaxRsClientV1Test extends HttpClientTest { Client client = Client.create() def setupSpec() { + client.setConnectTimeout(CONNECT_TIMEOUT_MS) + client.setReadTimeout(READ_TIMEOUT_MS) // Add filters to ensure spans aren't duplicated. client.addFilter(new LoggingFilter()) client.addFilter(new GZIPContentEncodingFilter()) diff --git a/dd-java-agent/instrumentation/jax-rs-client-2.0/src/test/groovy/JaxRsClientAsyncTest.groovy b/dd-java-agent/instrumentation/jax-rs-client-2.0/src/test/groovy/JaxRsClientAsyncTest.groovy index 65fcdfd5bc..d2d3e853c3 100644 --- a/dd-java-agent/instrumentation/jax-rs-client-2.0/src/test/groovy/JaxRsClientAsyncTest.groovy +++ b/dd-java-agent/instrumentation/jax-rs-client-2.0/src/test/groovy/JaxRsClientAsyncTest.groovy @@ -61,8 +61,8 @@ class JerseyClientAsyncTest extends JaxRsClientAsyncTest { @Override ClientBuilder builder() { ClientConfig config = new ClientConfig() - config.property(ClientProperties.CONNECT_TIMEOUT, 2000) - config.property(ClientProperties.READ_TIMEOUT, 2000) + config.property(ClientProperties.CONNECT_TIMEOUT, CONNECT_TIMEOUT_MS) + config.property(ClientProperties.READ_TIMEOUT, READ_TIMEOUT_MS) return new JerseyClientBuilder().withConfig(config) } @@ -77,8 +77,8 @@ class ResteasyClientAsyncTest extends JaxRsClientAsyncTest { @Override ClientBuilder builder() { return new ResteasyClientBuilder() - .establishConnectionTimeout(2, TimeUnit.SECONDS) - .socketTimeout(2, TimeUnit.SECONDS) + .establishConnectionTimeout(CONNECT_TIMEOUT_MS, TimeUnit.MILLISECONDS) + .socketTimeout(READ_TIMEOUT_MS, TimeUnit.MILLISECONDS) } boolean testRedirects() { diff --git a/dd-java-agent/instrumentation/jax-rs-client-2.0/src/test/groovy/JaxRsClientTest.groovy b/dd-java-agent/instrumentation/jax-rs-client-2.0/src/test/groovy/JaxRsClientTest.groovy index 65372a5c1a..6b7f544d37 100644 --- a/dd-java-agent/instrumentation/jax-rs-client-2.0/src/test/groovy/JaxRsClientTest.groovy +++ b/dd-java-agent/instrumentation/jax-rs-client-2.0/src/test/groovy/JaxRsClientTest.groovy @@ -51,8 +51,8 @@ class JerseyClientTest extends JaxRsClientTest { @Override ClientBuilder builder() { ClientConfig config = new ClientConfig() - config.property(ClientProperties.CONNECT_TIMEOUT, 2000) - config.property(ClientProperties.READ_TIMEOUT, 2000) + config.property(ClientProperties.CONNECT_TIMEOUT, CONNECT_TIMEOUT_MS) + config.property(ClientProperties.READ_TIMEOUT, READ_TIMEOUT_MS) return new JerseyClientBuilder().withConfig(config) } @@ -67,8 +67,8 @@ class ResteasyClientTest extends JaxRsClientTest { @Override ClientBuilder builder() { return new ResteasyClientBuilder() - .establishConnectionTimeout(2, TimeUnit.SECONDS) - .socketTimeout(2, TimeUnit.SECONDS) + .establishConnectionTimeout(CONNECT_TIMEOUT_MS, TimeUnit.MILLISECONDS) + .socketTimeout(READ_TIMEOUT_MS, TimeUnit.MILLISECONDS) } boolean testRedirects() { @@ -82,8 +82,8 @@ class CxfClientTest extends JaxRsClientTest { @Override ClientBuilder builder() { return new ClientBuilderImpl() -// .property(ClientImpl.HTTP_CONNECTION_TIMEOUT_PROP, 2000L) -// .property(ClientImpl.HTTP_RECEIVE_TIMEOUT_PROP, 2000L) +// .property(ClientImpl.HTTP_CONNECTION_TIMEOUT_PROP, (long) CONNECT_TIMEOUT_MS) +// .property(ClientImpl.HTTP_RECEIVE_TIMEOUT_PROP, (long) READ_TIMEOUT_MS) } boolean testRedirects() { diff --git a/dd-java-agent/instrumentation/okhttp-3/src/test/groovy/OkHttp3Test.groovy b/dd-java-agent/instrumentation/okhttp-3/src/test/groovy/OkHttp3Test.groovy index 7ff99a0b51..66e6dc9f7e 100644 --- a/dd-java-agent/instrumentation/okhttp-3/src/test/groovy/OkHttp3Test.groovy +++ b/dd-java-agent/instrumentation/okhttp-3/src/test/groovy/OkHttp3Test.groovy @@ -14,9 +14,9 @@ import java.util.concurrent.TimeUnit class OkHttp3Test extends HttpClientTest { def client = new OkHttpClient.Builder() - .connectTimeout(2, TimeUnit.SECONDS) - .readTimeout(2, TimeUnit.SECONDS) - .writeTimeout(2, TimeUnit.SECONDS) + .connectTimeout(CONNECT_TIMEOUT_MS, TimeUnit.MILLISECONDS) + .readTimeout(READ_TIMEOUT_MS, TimeUnit.MILLISECONDS) + .writeTimeout(READ_TIMEOUT_MS, TimeUnit.MILLISECONDS) .build() @Override diff --git a/dd-java-agent/instrumentation/vertx/src/test/groovy/client/VertxHttpClientTest.groovy b/dd-java-agent/instrumentation/vertx/src/test/groovy/client/VertxHttpClientTest.groovy index 738843db9b..68bcf1d42e 100644 --- a/dd-java-agent/instrumentation/vertx/src/test/groovy/client/VertxHttpClientTest.groovy +++ b/dd-java-agent/instrumentation/vertx/src/test/groovy/client/VertxHttpClientTest.groovy @@ -18,7 +18,7 @@ class VertxHttpClientTest extends HttpClientTest { @Shared def vertx = Vertx.vertx(new VertxOptions()) @Shared - def clientOptions = new HttpClientOptions().setConnectTimeout(2000).setIdleTimeout(2000) + def clientOptions = new HttpClientOptions().setConnectTimeout(CONNECT_TIMEOUT_MS).setIdleTimeout(READ_TIMEOUT_MS) @Shared def httpClient = vertx.createHttpClient(clientOptions) diff --git a/dd-java-agent/instrumentation/vertx/src/test/groovy/client/VertxRxCircuitBreakerWebClientTest.groovy b/dd-java-agent/instrumentation/vertx/src/test/groovy/client/VertxRxCircuitBreakerWebClientTest.groovy index 6b7fc721f6..2b69f4d9cb 100644 --- a/dd-java-agent/instrumentation/vertx/src/test/groovy/client/VertxRxCircuitBreakerWebClientTest.groovy +++ b/dd-java-agent/instrumentation/vertx/src/test/groovy/client/VertxRxCircuitBreakerWebClientTest.groovy @@ -20,7 +20,7 @@ class VertxRxCircuitBreakerWebClientTest extends HttpClientTest { @Shared Vertx vertx = Vertx.vertx(new VertxOptions()) @Shared - def clientOptions = new WebClientOptions().setConnectTimeout(2000).setIdleTimeout(2000) + def clientOptions = new WebClientOptions().setConnectTimeout(CONNECT_TIMEOUT_MS).setIdleTimeout(READ_TIMEOUT_MS) @Shared WebClient client = WebClient.create(vertx, clientOptions) @Shared diff --git a/dd-java-agent/instrumentation/vertx/src/test/groovy/client/VertxRxWebClientTest.groovy b/dd-java-agent/instrumentation/vertx/src/test/groovy/client/VertxRxWebClientTest.groovy index acde3648f7..0e7b549798 100644 --- a/dd-java-agent/instrumentation/vertx/src/test/groovy/client/VertxRxWebClientTest.groovy +++ b/dd-java-agent/instrumentation/vertx/src/test/groovy/client/VertxRxWebClientTest.groovy @@ -16,7 +16,7 @@ class VertxRxWebClientTest extends HttpClientTest { @Shared Vertx vertx = Vertx.vertx(new VertxOptions()) @Shared - def clientOptions = new WebClientOptions().setConnectTimeout(2000).setIdleTimeout(2000) + def clientOptions = new WebClientOptions().setConnectTimeout(CONNECT_TIMEOUT_MS).setIdleTimeout(READ_TIMEOUT_MS) @Shared WebClient client = WebClient.create(vertx, clientOptions) diff --git a/dd-java-agent/testing/src/main/groovy/datadog/trace/agent/test/base/HttpClientTest.groovy b/dd-java-agent/testing/src/main/groovy/datadog/trace/agent/test/base/HttpClientTest.groovy index 9132158770..c2d7b791a7 100644 --- a/dd-java-agent/testing/src/main/groovy/datadog/trace/agent/test/base/HttpClientTest.groovy +++ b/dd-java-agent/testing/src/main/groovy/datadog/trace/agent/test/base/HttpClientTest.groovy @@ -23,6 +23,8 @@ import static org.junit.Assume.assumeTrue @Unroll abstract class HttpClientTest extends AgentTestRunner { protected static final BODY_METHODS = ["POST", "PUT"] + protected static final CONNECT_TIMEOUT_MS = 1000 + protected static final READ_TIMEOUT_MS = 2000 @AutoCleanup @Shared From 7e6cb44ec6a1fbcac243eac116caa0161aa4945d Mon Sep 17 00:00:00 2001 From: Tyler Benson Date: Mon, 16 Mar 2020 09:40:33 -0700 Subject: [PATCH 03/43] More fixes --- .../groovy/ApacheHttpAsyncClientCallbackTest.groovy | 11 ++++++++--- .../ApacheHttpAsyncClientNullCallbackTest.groovy | 11 ++++++++--- .../src/test/groovy/ApacheHttpAsyncClientTest.groovy | 11 ++++++++--- .../trace/agent/test/base/HttpClientTest.groovy | 3 +++ .../testing/src/test/groovy/server/ServerTest.groovy | 4 ++-- 5 files changed, 29 insertions(+), 11 deletions(-) diff --git a/dd-java-agent/instrumentation/apache-httpasyncclient-4/src/test/groovy/ApacheHttpAsyncClientCallbackTest.groovy b/dd-java-agent/instrumentation/apache-httpasyncclient-4/src/test/groovy/ApacheHttpAsyncClientCallbackTest.groovy index d5248f6522..1b69580080 100644 --- a/dd-java-agent/instrumentation/apache-httpasyncclient-4/src/test/groovy/ApacheHttpAsyncClientCallbackTest.groovy +++ b/dd-java-agent/instrumentation/apache-httpasyncclient-4/src/test/groovy/ApacheHttpAsyncClientCallbackTest.groovy @@ -16,9 +16,9 @@ class ApacheHttpAsyncClientCallbackTest extends HttpClientTest { @Shared RequestConfig requestConfig = RequestConfig.custom() - .setSocketTimeout(1000) - .setConnectTimeout(1000) - .setConnectionRequestTimeout(1000).build() + .setConnectTimeout(CONNECT_TIMEOUT_MS) + .setSocketTimeout(READ_TIMEOUT_MS) + .build() @AutoCleanup @Shared @@ -68,4 +68,9 @@ class ApacheHttpAsyncClientCallbackTest extends HttpClientTest { Integer statusOnRedirectError() { return 302 } + + @Override + boolean testRemoteConnection() { + false // otherwise SocketTimeoutException for https requests + } } diff --git a/dd-java-agent/instrumentation/apache-httpasyncclient-4/src/test/groovy/ApacheHttpAsyncClientNullCallbackTest.groovy b/dd-java-agent/instrumentation/apache-httpasyncclient-4/src/test/groovy/ApacheHttpAsyncClientNullCallbackTest.groovy index 4cf58232d9..566403d37f 100644 --- a/dd-java-agent/instrumentation/apache-httpasyncclient-4/src/test/groovy/ApacheHttpAsyncClientNullCallbackTest.groovy +++ b/dd-java-agent/instrumentation/apache-httpasyncclient-4/src/test/groovy/ApacheHttpAsyncClientNullCallbackTest.groovy @@ -14,9 +14,9 @@ class ApacheHttpAsyncClientNullCallbackTest extends HttpClientTest { @Shared RequestConfig requestConfig = RequestConfig.custom() - .setSocketTimeout(1000) - .setConnectTimeout(1000) - .setConnectionRequestTimeout(1000).build() + .setConnectTimeout(CONNECT_TIMEOUT_MS) + .setSocketTimeout(READ_TIMEOUT_MS) + .build() @AutoCleanup @Shared @@ -54,4 +54,9 @@ class ApacheHttpAsyncClientNullCallbackTest extends HttpClientTest { Integer statusOnRedirectError() { return 302 } + + @Override + boolean testRemoteConnection() { + false // otherwise SocketTimeoutException for https requests + } } diff --git a/dd-java-agent/instrumentation/apache-httpasyncclient-4/src/test/groovy/ApacheHttpAsyncClientTest.groovy b/dd-java-agent/instrumentation/apache-httpasyncclient-4/src/test/groovy/ApacheHttpAsyncClientTest.groovy index 228c2fbcba..1330772a85 100644 --- a/dd-java-agent/instrumentation/apache-httpasyncclient-4/src/test/groovy/ApacheHttpAsyncClientTest.groovy +++ b/dd-java-agent/instrumentation/apache-httpasyncclient-4/src/test/groovy/ApacheHttpAsyncClientTest.groovy @@ -14,9 +14,9 @@ class ApacheHttpAsyncClientTest extends HttpClientTest { @Shared RequestConfig requestConfig = RequestConfig.custom() - .setSocketTimeout(1000) - .setConnectTimeout(1000) - .setConnectionRequestTimeout(1000).build() + .setConnectTimeout(CONNECT_TIMEOUT_MS) + .setSocketTimeout(READ_TIMEOUT_MS) + .build() @AutoCleanup @Shared @@ -63,4 +63,9 @@ class ApacheHttpAsyncClientTest extends HttpClientTest { Integer statusOnRedirectError() { return 302 } + + @Override + boolean testRemoteConnection() { + false // otherwise SocketTimeoutException for https requests + } } diff --git a/dd-java-agent/testing/src/main/groovy/datadog/trace/agent/test/base/HttpClientTest.groovy b/dd-java-agent/testing/src/main/groovy/datadog/trace/agent/test/base/HttpClientTest.groovy index c2d7b791a7..9eb3e60d11 100644 --- a/dd-java-agent/testing/src/main/groovy/datadog/trace/agent/test/base/HttpClientTest.groovy +++ b/dd-java-agent/testing/src/main/groovy/datadog/trace/agent/test/base/HttpClientTest.groovy @@ -8,6 +8,7 @@ import datadog.trace.api.DDSpanTypes import datadog.trace.api.DDTags import datadog.trace.bootstrap.instrumentation.api.Tags import spock.lang.AutoCleanup +import spock.lang.Requires import spock.lang.Shared import spock.lang.Unroll @@ -362,6 +363,8 @@ abstract class HttpClientTest extends AgentTestRunner { method = "HEAD" } + // IBM JVM has different protocol support for TLS + @Requires({ !System.getProperty("java.vm.name").contains("IBM J9 VM") }) def "test https request"() { given: assumeTrue(testRemoteConnection()) diff --git a/dd-java-agent/testing/src/test/groovy/server/ServerTest.groovy b/dd-java-agent/testing/src/test/groovy/server/ServerTest.groovy index 3c2109f329..5513c65bbb 100644 --- a/dd-java-agent/testing/src/test/groovy/server/ServerTest.groovy +++ b/dd-java-agent/testing/src/test/groovy/server/ServerTest.groovy @@ -44,13 +44,13 @@ class ServerTest extends AgentTestRunner { server.internalServer.isRunning() when: - server.stop() + server.close() then: !server.internalServer.isRunning() cleanup: - server.stop() + server.close() } def "server 404's with no handlers"() { From cf079a73ba46546f6538bc9ce47ae8420cc009fe Mon Sep 17 00:00:00 2001 From: Nikolay Martynov Date: Tue, 24 Mar 2020 11:37:19 -0400 Subject: [PATCH 04/43] Make sure that netty exception message is not null Otherwise netty gets really upset in an odd way --- .../groovy/Netty38ServerTest.groovy | 32 ++++++++++++++++--- .../src/test/groovy/Netty38ServerTest.groovy | 32 ++++++++++++++++--- 2 files changed, 54 insertions(+), 10 deletions(-) diff --git a/dd-java-agent/instrumentation/netty-3.8/src/latestDepTest/groovy/Netty38ServerTest.groovy b/dd-java-agent/instrumentation/netty-3.8/src/latestDepTest/groovy/Netty38ServerTest.groovy index e7b0794e0b..4416f948db 100644 --- a/dd-java-agent/instrumentation/netty-3.8/src/latestDepTest/groovy/Netty38ServerTest.groovy +++ b/dd-java-agent/instrumentation/netty-3.8/src/latestDepTest/groovy/Netty38ServerTest.groovy @@ -3,15 +3,36 @@ import datadog.trace.instrumentation.netty38.server.NettyHttpServerDecorator import org.jboss.netty.bootstrap.ServerBootstrap import org.jboss.netty.buffer.ChannelBuffer import org.jboss.netty.buffer.ChannelBuffers -import org.jboss.netty.channel.* +import org.jboss.netty.channel.Channel +import org.jboss.netty.channel.ChannelHandlerContext +import org.jboss.netty.channel.ChannelPipeline +import org.jboss.netty.channel.DefaultChannelPipeline +import org.jboss.netty.channel.DownstreamMessageEvent +import org.jboss.netty.channel.ExceptionEvent +import org.jboss.netty.channel.FailedChannelFuture +import org.jboss.netty.channel.MessageEvent +import org.jboss.netty.channel.SimpleChannelHandler +import org.jboss.netty.channel.SucceededChannelFuture import org.jboss.netty.channel.socket.nio.NioServerSocketChannelFactory -import org.jboss.netty.handler.codec.http.* +import org.jboss.netty.handler.codec.http.DefaultHttpResponse +import org.jboss.netty.handler.codec.http.HttpRequest +import org.jboss.netty.handler.codec.http.HttpResponse +import org.jboss.netty.handler.codec.http.HttpResponseStatus +import org.jboss.netty.handler.codec.http.HttpServerCodec import org.jboss.netty.handler.logging.LoggingHandler import org.jboss.netty.logging.InternalLogLevel import org.jboss.netty.util.CharsetUtil -import static datadog.trace.agent.test.base.HttpServerTest.ServerEndpoint.* -import static org.jboss.netty.handler.codec.http.HttpHeaders.Names.* +import static datadog.trace.agent.test.base.HttpServerTest.ServerEndpoint.ERROR +import static datadog.trace.agent.test.base.HttpServerTest.ServerEndpoint.EXCEPTION +import static datadog.trace.agent.test.base.HttpServerTest.ServerEndpoint.NOT_FOUND +import static datadog.trace.agent.test.base.HttpServerTest.ServerEndpoint.QUERY_PARAM +import static datadog.trace.agent.test.base.HttpServerTest.ServerEndpoint.REDIRECT +import static datadog.trace.agent.test.base.HttpServerTest.ServerEndpoint.SUCCESS +import static datadog.trace.agent.test.base.HttpServerTest.ServerEndpoint.forPath +import static org.jboss.netty.handler.codec.http.HttpHeaders.Names.CONTENT_LENGTH +import static org.jboss.netty.handler.codec.http.HttpHeaders.Names.CONTENT_TYPE +import static org.jboss.netty.handler.codec.http.HttpHeaders.Names.LOCATION import static org.jboss.netty.handler.codec.http.HttpVersion.HTTP_1_1 class Netty38ServerTest extends HttpServerTest { @@ -68,7 +89,8 @@ class Netty38ServerTest extends HttpServerTest { @Override void exceptionCaught(ChannelHandlerContext ctx, ExceptionEvent ex) throws Exception { - ChannelBuffer buffer = ChannelBuffers.copiedBuffer(ex.getCause().getMessage(), CharsetUtil.UTF_8) + def message = ex.getCause() == null ? " " + ex.message : ex.cause.message == null ? "" : ex.cause.message + ChannelBuffer buffer = ChannelBuffers.copiedBuffer(message, CharsetUtil.UTF_8) HttpResponse response = new DefaultHttpResponse(HTTP_1_1, HttpResponseStatus.INTERNAL_SERVER_ERROR) response.setContent(buffer) response.headers().set(CONTENT_TYPE, "text/plain") diff --git a/dd-java-agent/instrumentation/netty-3.8/src/test/groovy/Netty38ServerTest.groovy b/dd-java-agent/instrumentation/netty-3.8/src/test/groovy/Netty38ServerTest.groovy index e7b0794e0b..3d4982e1b1 100644 --- a/dd-java-agent/instrumentation/netty-3.8/src/test/groovy/Netty38ServerTest.groovy +++ b/dd-java-agent/instrumentation/netty-3.8/src/test/groovy/Netty38ServerTest.groovy @@ -3,15 +3,36 @@ import datadog.trace.instrumentation.netty38.server.NettyHttpServerDecorator import org.jboss.netty.bootstrap.ServerBootstrap import org.jboss.netty.buffer.ChannelBuffer import org.jboss.netty.buffer.ChannelBuffers -import org.jboss.netty.channel.* +import org.jboss.netty.channel.Channel +import org.jboss.netty.channel.ChannelHandlerContext +import org.jboss.netty.channel.ChannelPipeline +import org.jboss.netty.channel.DefaultChannelPipeline +import org.jboss.netty.channel.DownstreamMessageEvent +import org.jboss.netty.channel.ExceptionEvent +import org.jboss.netty.channel.FailedChannelFuture +import org.jboss.netty.channel.MessageEvent +import org.jboss.netty.channel.SimpleChannelHandler +import org.jboss.netty.channel.SucceededChannelFuture import org.jboss.netty.channel.socket.nio.NioServerSocketChannelFactory -import org.jboss.netty.handler.codec.http.* +import org.jboss.netty.handler.codec.http.DefaultHttpResponse +import org.jboss.netty.handler.codec.http.HttpRequest +import org.jboss.netty.handler.codec.http.HttpResponse +import org.jboss.netty.handler.codec.http.HttpResponseStatus +import org.jboss.netty.handler.codec.http.HttpServerCodec import org.jboss.netty.handler.logging.LoggingHandler import org.jboss.netty.logging.InternalLogLevel import org.jboss.netty.util.CharsetUtil -import static datadog.trace.agent.test.base.HttpServerTest.ServerEndpoint.* -import static org.jboss.netty.handler.codec.http.HttpHeaders.Names.* +import static datadog.trace.agent.test.base.HttpServerTest.ServerEndpoint.ERROR +import static datadog.trace.agent.test.base.HttpServerTest.ServerEndpoint.EXCEPTION +import static datadog.trace.agent.test.base.HttpServerTest.ServerEndpoint.NOT_FOUND +import static datadog.trace.agent.test.base.HttpServerTest.ServerEndpoint.QUERY_PARAM +import static datadog.trace.agent.test.base.HttpServerTest.ServerEndpoint.REDIRECT +import static datadog.trace.agent.test.base.HttpServerTest.ServerEndpoint.SUCCESS +import static datadog.trace.agent.test.base.HttpServerTest.ServerEndpoint.forPath +import static org.jboss.netty.handler.codec.http.HttpHeaders.Names.CONTENT_LENGTH +import static org.jboss.netty.handler.codec.http.HttpHeaders.Names.CONTENT_TYPE +import static org.jboss.netty.handler.codec.http.HttpHeaders.Names.LOCATION import static org.jboss.netty.handler.codec.http.HttpVersion.HTTP_1_1 class Netty38ServerTest extends HttpServerTest { @@ -68,7 +89,8 @@ class Netty38ServerTest extends HttpServerTest { @Override void exceptionCaught(ChannelHandlerContext ctx, ExceptionEvent ex) throws Exception { - ChannelBuffer buffer = ChannelBuffers.copiedBuffer(ex.getCause().getMessage(), CharsetUtil.UTF_8) + def message = ex.cause == null ? " " + ex.message : ex.cause.message == null ? "" : ex.cause.message + ChannelBuffer buffer = ChannelBuffers.copiedBuffer(message, CharsetUtil.UTF_8) HttpResponse response = new DefaultHttpResponse(HTTP_1_1, HttpResponseStatus.INTERNAL_SERVER_ERROR) response.setContent(buffer) response.headers().set(CONTENT_TYPE, "text/plain") From e1566889a4b6ea2ed6377546db6012f3a51a6abf Mon Sep 17 00:00:00 2001 From: Brian Devins-Suresh Date: Tue, 24 Mar 2020 13:49:38 -0400 Subject: [PATCH 05/43] Switch startup and shutdown impl --- .../groovy/Netty38ServerTest.groovy | 20 ++++++++++++------- .../src/test/groovy/Netty38ServerTest.groovy | 20 ++++++++++++------- 2 files changed, 26 insertions(+), 14 deletions(-) diff --git a/dd-java-agent/instrumentation/netty-3.8/src/latestDepTest/groovy/Netty38ServerTest.groovy b/dd-java-agent/instrumentation/netty-3.8/src/latestDepTest/groovy/Netty38ServerTest.groovy index 4416f948db..a3c300699c 100644 --- a/dd-java-agent/instrumentation/netty-3.8/src/latestDepTest/groovy/Netty38ServerTest.groovy +++ b/dd-java-agent/instrumentation/netty-3.8/src/latestDepTest/groovy/Netty38ServerTest.groovy @@ -3,9 +3,9 @@ import datadog.trace.instrumentation.netty38.server.NettyHttpServerDecorator import org.jboss.netty.bootstrap.ServerBootstrap import org.jboss.netty.buffer.ChannelBuffer import org.jboss.netty.buffer.ChannelBuffers -import org.jboss.netty.channel.Channel import org.jboss.netty.channel.ChannelHandlerContext import org.jboss.netty.channel.ChannelPipeline +import org.jboss.netty.channel.ChannelPipelineFactory import org.jboss.netty.channel.DefaultChannelPipeline import org.jboss.netty.channel.DownstreamMessageEvent import org.jboss.netty.channel.ExceptionEvent @@ -35,7 +35,7 @@ import static org.jboss.netty.handler.codec.http.HttpHeaders.Names.CONTENT_TYPE import static org.jboss.netty.handler.codec.http.HttpHeaders.Names.LOCATION import static org.jboss.netty.handler.codec.http.HttpVersion.HTTP_1_1 -class Netty38ServerTest extends HttpServerTest { +class Netty38ServerTest extends HttpServerTest { ChannelPipeline channelPipeline() { ChannelPipeline channelPipeline = new DefaultChannelPipeline() @@ -107,18 +107,24 @@ class Netty38ServerTest extends HttpServerTest { } @Override - Channel startServer(int port) { + ServerBootstrap startServer(int port) { ServerBootstrap bootstrap = new ServerBootstrap(new NioServerSocketChannelFactory()) bootstrap.setParentHandler(new LoggingHandler(InternalLogLevel.INFO)) - bootstrap.setPipeline(channelPipeline()) + bootstrap.setPipelineFactory(new ChannelPipelineFactory() { + @Override + ChannelPipeline getPipeline() throws Exception { + return channelPipeline() + } + }) InetSocketAddress address = new InetSocketAddress(port) - return bootstrap.bind(address) + bootstrap.bind(address) + return bootstrap } @Override - void stopServer(Channel server) { - server?.disconnect() + void stopServer(ServerBootstrap server) { + server?.shutdown() } @Override diff --git a/dd-java-agent/instrumentation/netty-3.8/src/test/groovy/Netty38ServerTest.groovy b/dd-java-agent/instrumentation/netty-3.8/src/test/groovy/Netty38ServerTest.groovy index 3d4982e1b1..7c1ba007e0 100644 --- a/dd-java-agent/instrumentation/netty-3.8/src/test/groovy/Netty38ServerTest.groovy +++ b/dd-java-agent/instrumentation/netty-3.8/src/test/groovy/Netty38ServerTest.groovy @@ -3,9 +3,9 @@ import datadog.trace.instrumentation.netty38.server.NettyHttpServerDecorator import org.jboss.netty.bootstrap.ServerBootstrap import org.jboss.netty.buffer.ChannelBuffer import org.jboss.netty.buffer.ChannelBuffers -import org.jboss.netty.channel.Channel import org.jboss.netty.channel.ChannelHandlerContext import org.jboss.netty.channel.ChannelPipeline +import org.jboss.netty.channel.ChannelPipelineFactory import org.jboss.netty.channel.DefaultChannelPipeline import org.jboss.netty.channel.DownstreamMessageEvent import org.jboss.netty.channel.ExceptionEvent @@ -35,7 +35,7 @@ import static org.jboss.netty.handler.codec.http.HttpHeaders.Names.CONTENT_TYPE import static org.jboss.netty.handler.codec.http.HttpHeaders.Names.LOCATION import static org.jboss.netty.handler.codec.http.HttpVersion.HTTP_1_1 -class Netty38ServerTest extends HttpServerTest { +class Netty38ServerTest extends HttpServerTest { ChannelPipeline channelPipeline() { ChannelPipeline channelPipeline = new DefaultChannelPipeline() @@ -107,18 +107,24 @@ class Netty38ServerTest extends HttpServerTest { } @Override - Channel startServer(int port) { + ServerBootstrap startServer(int port) { ServerBootstrap bootstrap = new ServerBootstrap(new NioServerSocketChannelFactory()) bootstrap.setParentHandler(new LoggingHandler(InternalLogLevel.INFO)) - bootstrap.setPipeline(channelPipeline()) + bootstrap.setPipelineFactory(new ChannelPipelineFactory() { + @Override + ChannelPipeline getPipeline() throws Exception { + return channelPipeline() + } + }) InetSocketAddress address = new InetSocketAddress(port) - return bootstrap.bind(address) + bootstrap.bind(address) + return bootstrap } @Override - void stopServer(Channel server) { - server?.disconnect() + void stopServer(ServerBootstrap server) { + server?.shutdown() } @Override From fee6d942e8cb6ae7234c8c18ba80430893a50a11 Mon Sep 17 00:00:00 2001 From: Nikolay Martynov Date: Tue, 24 Mar 2020 13:43:13 -0400 Subject: [PATCH 06/43] Add okhttp request logging in tests --- .../trace/agent/test/utils/OkHttpUtils.java | 24 +++++++++++++++++++ dd-java-agent/testing/testing.gradle | 1 + 2 files changed, 25 insertions(+) diff --git a/dd-java-agent/testing/src/main/groovy/datadog/trace/agent/test/utils/OkHttpUtils.java b/dd-java-agent/testing/src/main/groovy/datadog/trace/agent/test/utils/OkHttpUtils.java index e78633ed2d..46ef589cba 100644 --- a/dd-java-agent/testing/src/main/groovy/datadog/trace/agent/test/utils/OkHttpUtils.java +++ b/dd-java-agent/testing/src/main/groovy/datadog/trace/agent/test/utils/OkHttpUtils.java @@ -2,6 +2,10 @@ package datadog.trace.agent.test.utils; import java.util.concurrent.TimeUnit; import okhttp3.OkHttpClient; +import okhttp3.logging.HttpLoggingInterceptor; +import okhttp3.logging.HttpLoggingInterceptor.Level; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * This class was moved from groovy to java because groovy kept trying to introspect on the @@ -9,9 +13,29 @@ import okhttp3.OkHttpClient; */ public class OkHttpUtils { + private static final Logger CLIENT_LOGGER = LoggerFactory.getLogger("http-client"); + + static { + ((ch.qos.logback.classic.Logger) CLIENT_LOGGER).setLevel(ch.qos.logback.classic.Level.DEBUG); + } + + private static final HttpLoggingInterceptor LOGGING_INTERCEPTOR = + new HttpLoggingInterceptor( + new HttpLoggingInterceptor.Logger() { + @Override + public void log(final String message) { + CLIENT_LOGGER.debug(message); + } + }); + + static { + LOGGING_INTERCEPTOR.setLevel(Level.BASIC); + } + static OkHttpClient.Builder clientBuilder() { final TimeUnit unit = TimeUnit.MINUTES; return new OkHttpClient.Builder() + .addInterceptor(LOGGING_INTERCEPTOR) .connectTimeout(1, unit) .writeTimeout(1, unit) .readTimeout(1, unit); diff --git a/dd-java-agent/testing/testing.gradle b/dd-java-agent/testing/testing.gradle index 2d702c6aa4..ec45190746 100644 --- a/dd-java-agent/testing/testing.gradle +++ b/dd-java-agent/testing/testing.gradle @@ -22,6 +22,7 @@ dependencies { compile deps.guava compile group: 'org.eclipse.jetty', name: 'jetty-server', version: '8.0.0.v20110901' + compile group: 'com.squareup.okhttp3', name: 'logging-interceptor', version: versions.okhttp compile project(':dd-java-agent:agent-tooling') compile project(':utils:test-utils') From e5caa483642665e58cd01ce954b473c2ce463a55 Mon Sep 17 00:00:00 2001 From: Nikolay Martynov Date: Tue, 24 Mar 2020 14:51:26 -0400 Subject: [PATCH 07/43] Add netty server logging in tests --- .../src/latestDepTest/groovy/Netty38ServerTest.groovy | 11 ++++++++++- .../src/test/groovy/Netty38ServerTest.groovy | 11 ++++++++++- .../src/test/groovy/Netty40ServerTest.groovy | 6 +++++- .../src/test/groovy/Netty41ServerTest.groovy | 6 +++++- .../trace/agent/test/base/HttpServerTest.groovy | 8 ++++++++ 5 files changed, 38 insertions(+), 4 deletions(-) diff --git a/dd-java-agent/instrumentation/netty-3.8/src/latestDepTest/groovy/Netty38ServerTest.groovy b/dd-java-agent/instrumentation/netty-3.8/src/latestDepTest/groovy/Netty38ServerTest.groovy index a3c300699c..df2b5e050f 100644 --- a/dd-java-agent/instrumentation/netty-3.8/src/latestDepTest/groovy/Netty38ServerTest.groovy +++ b/dd-java-agent/instrumentation/netty-3.8/src/latestDepTest/groovy/Netty38ServerTest.groovy @@ -21,6 +21,8 @@ import org.jboss.netty.handler.codec.http.HttpResponseStatus import org.jboss.netty.handler.codec.http.HttpServerCodec import org.jboss.netty.handler.logging.LoggingHandler import org.jboss.netty.logging.InternalLogLevel +import org.jboss.netty.logging.InternalLoggerFactory +import org.jboss.netty.logging.Slf4JLoggerFactory import org.jboss.netty.util.CharsetUtil import static datadog.trace.agent.test.base.HttpServerTest.ServerEndpoint.ERROR @@ -37,8 +39,15 @@ import static org.jboss.netty.handler.codec.http.HttpVersion.HTTP_1_1 class Netty38ServerTest extends HttpServerTest { + static final LoggingHandler LOGGING_HANDLER + static { + InternalLoggerFactory.setDefaultFactory(new Slf4JLoggerFactory()) + LOGGING_HANDLER = new LoggingHandler(SERVER_LOGGER.name, InternalLogLevel.DEBUG, true) + } + ChannelPipeline channelPipeline() { ChannelPipeline channelPipeline = new DefaultChannelPipeline() + channelPipeline.addFirst("logger", LOGGING_HANDLER) channelPipeline.addLast("http-codec", new HttpServerCodec()) channelPipeline.addLast("controller", new SimpleChannelHandler() { @@ -109,7 +118,7 @@ class Netty38ServerTest extends HttpServerTest { @Override ServerBootstrap startServer(int port) { ServerBootstrap bootstrap = new ServerBootstrap(new NioServerSocketChannelFactory()) - bootstrap.setParentHandler(new LoggingHandler(InternalLogLevel.INFO)) + bootstrap.setParentHandler(LOGGING_HANDLER) bootstrap.setPipelineFactory(new ChannelPipelineFactory() { @Override ChannelPipeline getPipeline() throws Exception { diff --git a/dd-java-agent/instrumentation/netty-3.8/src/test/groovy/Netty38ServerTest.groovy b/dd-java-agent/instrumentation/netty-3.8/src/test/groovy/Netty38ServerTest.groovy index 7c1ba007e0..c08aa28a01 100644 --- a/dd-java-agent/instrumentation/netty-3.8/src/test/groovy/Netty38ServerTest.groovy +++ b/dd-java-agent/instrumentation/netty-3.8/src/test/groovy/Netty38ServerTest.groovy @@ -21,6 +21,8 @@ import org.jboss.netty.handler.codec.http.HttpResponseStatus import org.jboss.netty.handler.codec.http.HttpServerCodec import org.jboss.netty.handler.logging.LoggingHandler import org.jboss.netty.logging.InternalLogLevel +import org.jboss.netty.logging.InternalLoggerFactory +import org.jboss.netty.logging.Slf4JLoggerFactory import org.jboss.netty.util.CharsetUtil import static datadog.trace.agent.test.base.HttpServerTest.ServerEndpoint.ERROR @@ -37,8 +39,15 @@ import static org.jboss.netty.handler.codec.http.HttpVersion.HTTP_1_1 class Netty38ServerTest extends HttpServerTest { + static final LoggingHandler LOGGING_HANDLER + static { + InternalLoggerFactory.setDefaultFactory(new Slf4JLoggerFactory()) + LOGGING_HANDLER = new LoggingHandler(SERVER_LOGGER.name, InternalLogLevel.DEBUG, true) + } + ChannelPipeline channelPipeline() { ChannelPipeline channelPipeline = new DefaultChannelPipeline() + channelPipeline.addFirst("logger", LOGGING_HANDLER) channelPipeline.addLast("http-codec", new HttpServerCodec()) channelPipeline.addLast("controller", new SimpleChannelHandler() { @@ -109,7 +118,7 @@ class Netty38ServerTest extends HttpServerTest { @Override ServerBootstrap startServer(int port) { ServerBootstrap bootstrap = new ServerBootstrap(new NioServerSocketChannelFactory()) - bootstrap.setParentHandler(new LoggingHandler(InternalLogLevel.INFO)) + bootstrap.setParentHandler(LOGGING_HANDLER) bootstrap.setPipelineFactory(new ChannelPipelineFactory() { @Override ChannelPipeline getPipeline() throws Exception { diff --git a/dd-java-agent/instrumentation/netty-4.0/src/test/groovy/Netty40ServerTest.groovy b/dd-java-agent/instrumentation/netty-4.0/src/test/groovy/Netty40ServerTest.groovy index d7affb8e11..e97652e43f 100644 --- a/dd-java-agent/instrumentation/netty-4.0/src/test/groovy/Netty40ServerTest.groovy +++ b/dd-java-agent/instrumentation/netty-4.0/src/test/groovy/Netty40ServerTest.groovy @@ -34,16 +34,20 @@ import static io.netty.handler.codec.http.HttpVersion.HTTP_1_1 class Netty40ServerTest extends HttpServerTest { + static final LoggingHandler LOGGING_HANDLER = new LoggingHandler(SERVER_LOGGER.name, LogLevel.DEBUG) + @Override EventLoopGroup startServer(int port) { def eventLoopGroup = new NioEventLoopGroup() ServerBootstrap bootstrap = new ServerBootstrap() .group(eventLoopGroup) - .handler(new LoggingHandler(LogLevel.INFO)) + .handler(LOGGING_HANDLER) .childHandler([ initChannel: { ch -> ChannelPipeline pipeline = ch.pipeline() + pipeline.addFirst("logger", LOGGING_HANDLER) + def handlers = [new HttpRequestDecoder(), new HttpResponseEncoder()] handlers.each { pipeline.addLast(it) } pipeline.addLast([ diff --git a/dd-java-agent/instrumentation/netty-4.1/src/test/groovy/Netty41ServerTest.groovy b/dd-java-agent/instrumentation/netty-4.1/src/test/groovy/Netty41ServerTest.groovy index d3dcf3d433..a91e779f28 100644 --- a/dd-java-agent/instrumentation/netty-4.1/src/test/groovy/Netty41ServerTest.groovy +++ b/dd-java-agent/instrumentation/netty-4.1/src/test/groovy/Netty41ServerTest.groovy @@ -33,16 +33,20 @@ import static io.netty.handler.codec.http.HttpVersion.HTTP_1_1 class Netty41ServerTest extends HttpServerTest { + static final LoggingHandler LOGGING_HANDLER = new LoggingHandler(SERVER_LOGGER.name, LogLevel.DEBUG) + @Override EventLoopGroup startServer(int port) { def eventLoopGroup = new NioEventLoopGroup() ServerBootstrap bootstrap = new ServerBootstrap() .group(eventLoopGroup) - .handler(new LoggingHandler(LogLevel.INFO)) + .handler(LOGGING_HANDLER) .childHandler([ initChannel: { ch -> ChannelPipeline pipeline = ch.pipeline() + pipeline.addFirst("logger", LOGGING_HANDLER) + def handlers = [new HttpServerCodec()] handlers.each { pipeline.addLast(it) } pipeline.addLast([ diff --git a/dd-java-agent/testing/src/main/groovy/datadog/trace/agent/test/base/HttpServerTest.groovy b/dd-java-agent/testing/src/main/groovy/datadog/trace/agent/test/base/HttpServerTest.groovy index 13f25c2c4c..16990954b1 100644 --- a/dd-java-agent/testing/src/main/groovy/datadog/trace/agent/test/base/HttpServerTest.groovy +++ b/dd-java-agent/testing/src/main/groovy/datadog/trace/agent/test/base/HttpServerTest.groovy @@ -1,5 +1,6 @@ package datadog.trace.agent.test.base +import ch.qos.logback.classic.Level import datadog.opentracing.DDSpan import datadog.trace.agent.test.AgentTestRunner import datadog.trace.agent.test.asserts.ListWriterAssert @@ -15,6 +16,8 @@ import okhttp3.HttpUrl import okhttp3.OkHttpClient import okhttp3.Request import okhttp3.Response +import org.slf4j.Logger +import org.slf4j.LoggerFactory import spock.lang.Shared import spock.lang.Unroll @@ -37,6 +40,11 @@ import static org.junit.Assume.assumeTrue @Unroll abstract class HttpServerTest extends AgentTestRunner { + public static final Logger SERVER_LOGGER = LoggerFactory.getLogger("http-server") + static { + ((ch.qos.logback.classic.Logger) SERVER_LOGGER).setLevel(Level.DEBUG) + } + @Shared SERVER server @Shared From cda39adf50bc5ec9b74117292dc0e67332b4777a Mon Sep 17 00:00:00 2001 From: Lev Priima <1118651+lpriima@users.noreply.github.com> Date: Wed, 25 Mar 2020 11:17:47 -0700 Subject: [PATCH 08/43] add ..."metrics":{"_dd.measured":1}... to every span (#1336) * add ..."metrics":{"_dd.measured":1}... to every span * update tests to always have "metrics":{"_dd.measured":1} in every span Authored-by: Lev Priima --- .../src/main/java/datadog/trace/api/Config.java | 2 +- .../main/java/datadog/opentracing/DDSpanContext.java | 9 ++++++--- .../opentracing/DDSpanSerializationTest.groovy | 1 + .../opentracing/decorators/SpanDecoratorTest.groovy | 5 ++++- .../groovy/datadog/trace/DDSpanContextTest.groovy | 11 ++++++++--- .../datadog/trace/api/writer/DDAgentApiTest.groovy | 6 ++++-- .../datadog/trace/api/writer/LoggingWriterTest.groovy | 2 +- 7 files changed, 25 insertions(+), 11 deletions(-) diff --git a/dd-trace-api/src/main/java/datadog/trace/api/Config.java b/dd-trace-api/src/main/java/datadog/trace/api/Config.java index a53f58c2d0..e9b6e9c816 100644 --- a/dd-trace-api/src/main/java/datadog/trace/api/Config.java +++ b/dd-trace-api/src/main/java/datadog/trace/api/Config.java @@ -837,7 +837,7 @@ public class Config { * @return * @deprecated This method should only be used internally. Use the explicit getter instead. */ - public static String getSettingFromEnvironment(final String name, final String defaultValue) { + private static String getSettingFromEnvironment(final String name, final String defaultValue) { String value; // System properties and properties provided from command line have the highest precedence diff --git a/dd-trace-ot/src/main/java/datadog/opentracing/DDSpanContext.java b/dd-trace-ot/src/main/java/datadog/opentracing/DDSpanContext.java index 88e0974135..38ad5535d4 100644 --- a/dd-trace-ot/src/main/java/datadog/opentracing/DDSpanContext.java +++ b/dd-trace-ot/src/main/java/datadog/opentracing/DDSpanContext.java @@ -25,8 +25,11 @@ public class DDSpanContext implements io.opentracing.SpanContext { public static final String PRIORITY_SAMPLING_KEY = "_sampling_priority_v1"; public static final String SAMPLE_RATE_KEY = "_sample_rate"; public static final String ORIGIN_KEY = "_dd.origin"; + public static final String DD_MEASURED = "_dd.measured"; + public static final Number DD_MEASURED_DEFAULT = 1; - private static final Map EMPTY_METRICS = Collections.emptyMap(); + private static final Map DEFAULT_METRICS = + Collections.singletonMap(DD_MEASURED, DD_MEASURED_DEFAULT); // Shared with other span contexts /** For technical reasons, the ref to the original tracer */ @@ -307,12 +310,12 @@ public class DDSpanContext implements io.opentracing.SpanContext { public Map getMetrics() { final Map metrics = this.metrics.get(); - return metrics == null ? EMPTY_METRICS : metrics; + return metrics == null ? DEFAULT_METRICS : metrics; } public void setMetric(final String key, final Number value) { if (metrics.get() == null) { - metrics.compareAndSet(null, new ConcurrentHashMap()); + metrics.compareAndSet(null, new ConcurrentHashMap<>(DEFAULT_METRICS)); } if (value instanceof Float) { metrics.get().put(key, value.doubleValue()); diff --git a/dd-trace-ot/src/test/groovy/datadog/opentracing/DDSpanSerializationTest.groovy b/dd-trace-ot/src/test/groovy/datadog/opentracing/DDSpanSerializationTest.groovy index fa8d46580a..89f1c00742 100644 --- a/dd-trace-ot/src/test/groovy/datadog/opentracing/DDSpanSerializationTest.groovy +++ b/dd-trace-ot/src/test/groovy/datadog/opentracing/DDSpanSerializationTest.groovy @@ -20,6 +20,7 @@ class DDSpanSerializationTest extends DDSpecification { def jsonAdapter = new Moshi.Builder().build().adapter(Map) final Map metrics = ["_sampling_priority_v1": 1] + metrics.putAll(DDSpanContext.DEFAULT_METRICS) if (samplingPriority == PrioritySampling.UNSET) { // RateByServiceSampler sets priority metrics.put("_dd.agent_psr", 1.0d) } diff --git a/dd-trace-ot/src/test/groovy/datadog/opentracing/decorators/SpanDecoratorTest.groovy b/dd-trace-ot/src/test/groovy/datadog/opentracing/decorators/SpanDecoratorTest.groovy index 094b5d77ce..8725967f40 100644 --- a/dd-trace-ot/src/test/groovy/datadog/opentracing/decorators/SpanDecoratorTest.groovy +++ b/dd-trace-ot/src/test/groovy/datadog/opentracing/decorators/SpanDecoratorTest.groovy @@ -279,10 +279,13 @@ class SpanDecoratorTest extends DDSpecification { def "span metrics starts empty but added with rate limiting value of #rate"() { expect: - span.metrics == [:] + span.metrics == DDSpanContext.DEFAULT_METRICS when: span.setTag(ANALYTICS_SAMPLE_RATE, rate) + // these 2 lines to avoid checking for {@code DDSpanContext.DD_MEASURED} for every metric: + span.context().setMetric(DDSpanContext.DD_MEASURED, 42) + span.metrics.remove(DDSpanContext.DD_MEASURED, 42) then: span.metrics == result diff --git a/dd-trace-ot/src/test/groovy/datadog/trace/DDSpanContextTest.groovy b/dd-trace-ot/src/test/groovy/datadog/trace/DDSpanContextTest.groovy index ce2b423829..55ac7a945b 100644 --- a/dd-trace-ot/src/test/groovy/datadog/trace/DDSpanContextTest.groovy +++ b/dd-trace-ot/src/test/groovy/datadog/trace/DDSpanContextTest.groovy @@ -1,5 +1,6 @@ package datadog.trace +import datadog.opentracing.DDSpanContext import datadog.opentracing.SpanFactory import datadog.trace.api.DDTags import datadog.trace.util.test.DDSpecification @@ -18,7 +19,7 @@ class DDSpanContextTest extends DDSpecification { context.serviceName == "fakeService" context.resourceName == "fakeResource" context.spanType == "fakeType" - context.toString() == "DDSpan [ t_id=1, s_id=1, p_id=0] trace=fakeService/fakeOperation/fakeResource metrics={} *errored* tags={${extra}${tags.containsKey(DDTags.SPAN_TYPE) ? "span.type=${context.getSpanType()}, " : ""}thread.id=${Thread.currentThread().id}, thread.name=${Thread.currentThread().name}}" + context.toString() == "DDSpan [ t_id=1, s_id=1, p_id=0] trace=fakeService/fakeOperation/fakeResource metrics=${defaultMetrics()} *errored* tags={${extra}${tags.containsKey(DDTags.SPAN_TYPE) ? "span.type=${context.getSpanType()}, " : ""}thread.id=${Thread.currentThread().id}, thread.name=${Thread.currentThread().name}}" where: name | extra | tags @@ -35,7 +36,7 @@ class DDSpanContextTest extends DDSpecification { def thread = Thread.currentThread() def expectedTags = [(DDTags.THREAD_NAME): thread.name, (DDTags.THREAD_ID): thread.id] - def expectedTrace = "DDSpan [ t_id=1, s_id=1, p_id=0] trace=$details metrics={} tags={thread.id=$thread.id, thread.name=$thread.name}" + def expectedTrace = "DDSpan [ t_id=1, s_id=1, p_id=0] trace=$details metrics=${defaultMetrics()} tags={thread.id=$thread.id, thread.name=$thread.name}" expect: context.getTags() == expectedTags @@ -61,7 +62,7 @@ class DDSpanContextTest extends DDSpecification { (DDTags.THREAD_NAME): thread.name, (DDTags.THREAD_ID) : thread.id ] - context.toString() == "DDSpan [ t_id=1, s_id=1, p_id=0] trace=fakeService/fakeOperation/fakeResource metrics={} tags={$name=$value, thread.id=$thread.id, thread.name=$thread.name}" + context.toString() == "DDSpan [ t_id=1, s_id=1, p_id=0] trace=fakeService/fakeOperation/fakeResource metrics=${defaultMetrics()} tags={$name=$value, thread.id=$thread.id, thread.name=$thread.name}" where: name | value @@ -98,4 +99,8 @@ class DDSpanContextTest extends DDSpecification { Double | 0.5d Integer | 0x55 } + + static String defaultMetrics() { + return DDSpanContext.DEFAULT_METRICS + } } diff --git a/dd-trace-ot/src/test/groovy/datadog/trace/api/writer/DDAgentApiTest.groovy b/dd-trace-ot/src/test/groovy/datadog/trace/api/writer/DDAgentApiTest.groovy index 189dca2664..6cff04562d 100644 --- a/dd-trace-ot/src/test/groovy/datadog/trace/api/writer/DDAgentApiTest.groovy +++ b/dd-trace-ot/src/test/groovy/datadog/trace/api/writer/DDAgentApiTest.groovy @@ -2,6 +2,7 @@ package datadog.trace.api.writer import com.fasterxml.jackson.core.type.TypeReference import com.fasterxml.jackson.databind.ObjectMapper +import datadog.opentracing.DDSpanContext import datadog.opentracing.SpanFactory import datadog.trace.common.writer.ddagent.DDAgentApi import datadog.trace.common.writer.ddagent.DDAgentResponseListener @@ -101,7 +102,8 @@ class DDAgentApiTest extends DDSpecification { "duration" : 0, "error" : 0, "meta" : ["thread.name": Thread.currentThread().getName(), "thread.id": "${Thread.currentThread().id}"], - "metrics" : [:], + //TODO : use DDSpanContext.DD_MEASURED as a key + "metrics" : ["_dd.measured": DDSpanContext.DD_MEASURED_DEFAULT], "name" : "fakeOperation", "parent_id": 0, "resource" : "fakeResource", @@ -115,7 +117,7 @@ class DDAgentApiTest extends DDSpecification { "duration" : 0, "error" : 0, "meta" : ["thread.name": Thread.currentThread().getName(), "thread.id": "${Thread.currentThread().id}"], - "metrics" : [:], + "metrics" : ["_dd.measured": DDSpanContext.DD_MEASURED_DEFAULT], "name" : "fakeOperation", "parent_id": 0, "resource" : "my-resource", diff --git a/dd-trace-ot/src/test/groovy/datadog/trace/api/writer/LoggingWriterTest.groovy b/dd-trace-ot/src/test/groovy/datadog/trace/api/writer/LoggingWriterTest.groovy index 63cacf0684..b981784a26 100644 --- a/dd-trace-ot/src/test/groovy/datadog/trace/api/writer/LoggingWriterTest.groovy +++ b/dd-trace-ot/src/test/groovy/datadog/trace/api/writer/LoggingWriterTest.groovy @@ -16,6 +16,6 @@ class LoggingWriterTest extends DDSpecification { def "test toString"() { expect: - writer.toString(sampleTrace).startsWith('[{"service":"fakeService","name":"fakeOperation","resource":"fakeResource","trace_id":1,"span_id":1,"parent_id":0,"start":1000,"duration":0,"type":"fakeType","error":0,"metrics":{},"meta":{') + writer.toString(sampleTrace).startsWith('[{"service":"fakeService","name":"fakeOperation","resource":"fakeResource","trace_id":1,"span_id":1,"parent_id":0,"start":1000,"duration":0,"type":"fakeType","error":0,"metrics":{"_dd.measured":1},"meta":{') } } From 4ac364d92aa2de8940e21d1706746788d19be3b7 Mon Sep 17 00:00:00 2001 From: Brian Devins-Suresh Date: Wed, 25 Mar 2020 16:29:01 -0400 Subject: [PATCH 09/43] Fix tests --- .../src/latestDepTest/groovy/Netty38ClientTest.groovy | 5 +++++ .../netty-3.8/src/test/groovy/Netty38ClientTest.groovy | 7 ++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/dd-java-agent/instrumentation/netty-3.8/src/latestDepTest/groovy/Netty38ClientTest.groovy b/dd-java-agent/instrumentation/netty-3.8/src/latestDepTest/groovy/Netty38ClientTest.groovy index 9c26e09b90..afacb7ca70 100644 --- a/dd-java-agent/instrumentation/netty-3.8/src/latestDepTest/groovy/Netty38ClientTest.groovy +++ b/dd-java-agent/instrumentation/netty-3.8/src/latestDepTest/groovy/Netty38ClientTest.groovy @@ -62,6 +62,11 @@ class Netty38ClientTest extends HttpClientTest { false } + @Override + boolean testRemoteConnection() { + return false + } + def "connection error (unopened port)"() { given: def uri = new URI("http://localhost:$UNUSABLE_PORT/") diff --git a/dd-java-agent/instrumentation/netty-3.8/src/test/groovy/Netty38ClientTest.groovy b/dd-java-agent/instrumentation/netty-3.8/src/test/groovy/Netty38ClientTest.groovy index c56d227845..761f2589b4 100644 --- a/dd-java-agent/instrumentation/netty-3.8/src/test/groovy/Netty38ClientTest.groovy +++ b/dd-java-agent/instrumentation/netty-3.8/src/test/groovy/Netty38ClientTest.groovy @@ -21,7 +21,7 @@ class Netty38ClientTest extends HttpClientTest { def clientConfig = new AsyncHttpClientConfig.Builder() .setRequestTimeoutInMs(TimeUnit.SECONDS.toMillis(10).toInteger()) .build() - + @Shared @AutoCleanup AsyncHttpClient asyncHttpClient = new AsyncHttpClient(clientConfig) @@ -62,6 +62,11 @@ class Netty38ClientTest extends HttpClientTest { false } + @Override + boolean testRemoteConnection() { + return false + } + def "connection error (unopened port)"() { given: def uri = new URI("http://localhost:$UNUSABLE_PORT/") From b63d0c68261e23610275041b58aba22e03b0401e Mon Sep 17 00:00:00 2001 From: "Amine.Mesbahi" Date: Thu, 26 Mar 2020 10:58:48 +0100 Subject: [PATCH 10/43] Add copy_artifact script --- .circleci/copy_artifact_s3.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 .circleci/copy_artifact_s3.py diff --git a/.circleci/copy_artifact_s3.py b/.circleci/copy_artifact_s3.py new file mode 100644 index 0000000000..08f37028c7 --- /dev/null +++ b/.circleci/copy_artifact_s3.py @@ -0,0 +1,23 @@ +import boto3 +import re +import os +import tempfile + +LIBS_PATH = './workspace/dd-java-agent/build/libs' +p = re.compile(r"dd-java-agent.*jar$") + +S3_BUCKET_NAME = 'datadog-reliability-env' +client = boto3.client('s3', aws_access_key_id=os.getenv('AWS_ACCESS_KEY_ID'),aws_secret_access_key=os.getenv('AWS_SECRET_ACCESS_KEY')) +transfer = boto3.s3.transfer.S3Transfer(client) + +for path, sub_dirs, files in os.walk(LIBS_PATH): + for name in files: + if p.match(name): + # Write the artifact to S3 + transfer.upload_file(os.path.join(path, name), S3_BUCKET_NAME, f'java/{name}') + # write additional information used by the build + with tempfile.NamedTemporaryFile(mode='w') as fp: + for line in [os.getenv('CIRCLE_BRANCH'), os.getenv('CIRCLE_SHA1'), name, os.getenv('CIRCLE_USERNAME')]: + fp.write(f'{line}\n') + fp.seek(0) + transfer.upload_file(fp.name, S3_BUCKET_NAME, 'java/index.txt') From 4a30faa8ffbacc916d2909af7c8b53ae5c25f32d Mon Sep 17 00:00:00 2001 From: "Amine.Mesbahi" Date: Thu, 26 Mar 2020 11:18:30 +0100 Subject: [PATCH 11/43] Add copy_artifact script to config --- .circleci/config.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 6679cccbbc..cdce5c73e4 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -252,6 +252,18 @@ jobs: publish_tag: <<: *publish + copy_artifact_s3: + docker: + - image: circleci/python:3.6.4 + steps: + - checkout + - attach_workspace: + at: . + - run: + command: | + sudo pip install boto3 + python .circleci/copy_artifact_s3.py + workflows: version: 2 build_test_deploy: @@ -341,6 +353,10 @@ workflows: tags: only: /.*/ + - copy_artifact_s3: + requires: + - build + - check: requires: - build From 4083212814b3bf2b85aeeae6dbb821fc5f39c643 Mon Sep 17 00:00:00 2001 From: "Amine.Mesbahi" Date: Thu, 26 Mar 2020 13:44:57 +0100 Subject: [PATCH 12/43] Apply copy artifact to only master and *-reliability --- .circleci/config.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index cdce5c73e4..305ebf57ad 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -356,6 +356,11 @@ workflows: - copy_artifact_s3: requires: - build + filters: + branches: + only: + - master + - .*-reliability - check: requires: From 73122ea72c563c9ed4b5b0bd604251be766f250b Mon Sep 17 00:00:00 2001 From: Brian Devins-Suresh Date: Mon, 16 Mar 2020 16:55:31 -0400 Subject: [PATCH 13/43] Starting to get tests building --- .../instrumentation/play-2.3/.gitignore | 1 + .../instrumentation/play-2.3/play-2.3.gradle | 47 ++++++++++ .../play24/PlayInstrumentation.java | 52 +++++++++++ .../instrumentation/play24/PlayAdvice.java | 73 ++++++++++++++++ .../instrumentation/play24/PlayHeaders.java | 26 ++++++ .../play24/PlayHttpServerDecorator.java | 86 +++++++++++++++++++ .../play24/RequestCompleteCallback.java | 40 +++++++++ .../groovy/client/PlayWSClientTest.groovy | 54 ++++++++++++ .../NettyServerTestInstrumentation.java | 22 +++++ .../groovy/server/PlayAsyncServerTest.groovy | 46 ++++++++++ .../test/groovy/server/PlayServerTest.groovy | 79 +++++++++++++++++ .../src/test/scala/server/AsyncServer.scala | 5 ++ .../src/test/scala/server/SyncServer.scala | 15 ++++ settings.gradle | 1 + 14 files changed, 547 insertions(+) create mode 100644 dd-java-agent/instrumentation/play-2.3/.gitignore create mode 100644 dd-java-agent/instrumentation/play-2.3/play-2.3.gradle create mode 100644 dd-java-agent/instrumentation/play-2.3/src/main/java/datadog/trace/instrumentation/play24/PlayInstrumentation.java create mode 100644 dd-java-agent/instrumentation/play-2.3/src/main/java8/datadog/trace/instrumentation/play24/PlayAdvice.java create mode 100644 dd-java-agent/instrumentation/play-2.3/src/main/java8/datadog/trace/instrumentation/play24/PlayHeaders.java create mode 100644 dd-java-agent/instrumentation/play-2.3/src/main/java8/datadog/trace/instrumentation/play24/PlayHttpServerDecorator.java create mode 100644 dd-java-agent/instrumentation/play-2.3/src/main/java8/datadog/trace/instrumentation/play24/RequestCompleteCallback.java create mode 100644 dd-java-agent/instrumentation/play-2.3/src/test/groovy/client/PlayWSClientTest.groovy create mode 100644 dd-java-agent/instrumentation/play-2.3/src/test/groovy/server/NettyServerTestInstrumentation.java create mode 100644 dd-java-agent/instrumentation/play-2.3/src/test/groovy/server/PlayAsyncServerTest.groovy create mode 100644 dd-java-agent/instrumentation/play-2.3/src/test/groovy/server/PlayServerTest.groovy create mode 100644 dd-java-agent/instrumentation/play-2.3/src/test/scala/server/AsyncServer.scala create mode 100644 dd-java-agent/instrumentation/play-2.3/src/test/scala/server/SyncServer.scala diff --git a/dd-java-agent/instrumentation/play-2.3/.gitignore b/dd-java-agent/instrumentation/play-2.3/.gitignore new file mode 100644 index 0000000000..5292519a25 --- /dev/null +++ b/dd-java-agent/instrumentation/play-2.3/.gitignore @@ -0,0 +1 @@ +logs/ \ No newline at end of file diff --git a/dd-java-agent/instrumentation/play-2.3/play-2.3.gradle b/dd-java-agent/instrumentation/play-2.3/play-2.3.gradle new file mode 100644 index 0000000000..e77bad0dea --- /dev/null +++ b/dd-java-agent/instrumentation/play-2.3/play-2.3.gradle @@ -0,0 +1,47 @@ +ext { + minJavaVersionForTests = JavaVersion.VERSION_1_8 + // Play doesn't work with Java 9+ until 2.6.12 + maxJavaVersionForTests = JavaVersion.VERSION_1_8 +} + +muzzle { + pass { + group = 'com.typesafe.play' + module = 'play_2.11' + versions = '[2.3.0,2.4)' + assertInverse = true + } + fail { + group = 'com.typesafe.play' + module = 'play_2.12' + versions = '[,]' + } + fail { + group = 'com.typesafe.play' + module = 'play_2.13' + versions = '[,]' + } +} + +apply from: "${rootDir}/gradle/java.gradle" +apply from: "${rootDir}/gradle/test-with-scala.gradle" + +apply plugin: 'org.unbroken-dome.test-sets' + +testSets { + latestDepTest { + dirName = 'test' + } +} + +dependencies { + main_java8Compile group: 'com.typesafe.play', name: 'play_2.11', version: '2.3.9' + + testCompile group: 'com.typesafe.play', name: 'play-java_2.11', version: '2.3.9' + testCompile group: 'com.typesafe.play', name: 'play-java-ws_2.11', version: '2.3.9' + testCompile(group: 'com.typesafe.play', name: 'play-test_2.11', version: '2.3.9') + + latestDepTestCompile group: 'com.typesafe.play', name: 'play-java_2.11', version: '2.3.+' + latestDepTestCompile group: 'com.typesafe.play', name: 'play-java-ws_2.11', version: '2.3.+' + latestDepTestCompile(group: 'com.typesafe.play', name: 'play-test_2.11', version: '2.3.+') +} diff --git a/dd-java-agent/instrumentation/play-2.3/src/main/java/datadog/trace/instrumentation/play24/PlayInstrumentation.java b/dd-java-agent/instrumentation/play-2.3/src/main/java/datadog/trace/instrumentation/play24/PlayInstrumentation.java new file mode 100644 index 0000000000..fa7d265158 --- /dev/null +++ b/dd-java-agent/instrumentation/play-2.3/src/main/java/datadog/trace/instrumentation/play24/PlayInstrumentation.java @@ -0,0 +1,52 @@ +package datadog.trace.instrumentation.play24; + +import static datadog.trace.agent.tooling.ClassLoaderMatcher.hasClassesNamed; +import static datadog.trace.agent.tooling.bytebuddy.matcher.DDElementMatchers.implementsInterface; +import static java.util.Collections.singletonMap; +import static net.bytebuddy.matcher.ElementMatchers.named; +import static net.bytebuddy.matcher.ElementMatchers.returns; +import static net.bytebuddy.matcher.ElementMatchers.takesArgument; + +import com.google.auto.service.AutoService; +import datadog.trace.agent.tooling.Instrumenter; +import java.util.Map; +import net.bytebuddy.description.method.MethodDescription; +import net.bytebuddy.description.type.TypeDescription; +import net.bytebuddy.matcher.ElementMatcher; + +@AutoService(Instrumenter.class) +public final class PlayInstrumentation extends Instrumenter.Default { + + public PlayInstrumentation() { + super("play"); + } + + @Override + public ElementMatcher classLoaderMatcher() { + // Optimization for expensive typeMatcher. + return hasClassesNamed("play.api.mvc.Action"); + } + + @Override + public ElementMatcher typeMatcher() { + return implementsInterface(named("play.api.mvc.Action")); + } + + @Override + public String[] helperClassNames() { + return new String[] { + packageName + ".PlayHttpServerDecorator", + packageName + ".RequestCompleteCallback", + packageName + ".PlayHeaders", + }; + } + + @Override + public Map, String> transformers() { + return singletonMap( + named("apply") + .and(takesArgument(0, named("play.api.mvc.Request"))) + .and(returns(named("scala.concurrent.Future"))), + packageName + ".PlayAdvice"); + } +} diff --git a/dd-java-agent/instrumentation/play-2.3/src/main/java8/datadog/trace/instrumentation/play24/PlayAdvice.java b/dd-java-agent/instrumentation/play-2.3/src/main/java8/datadog/trace/instrumentation/play24/PlayAdvice.java new file mode 100644 index 0000000000..2e495f2e72 --- /dev/null +++ b/dd-java-agent/instrumentation/play-2.3/src/main/java8/datadog/trace/instrumentation/play24/PlayAdvice.java @@ -0,0 +1,73 @@ +package datadog.trace.instrumentation.play24; + +import static datadog.trace.bootstrap.instrumentation.api.AgentTracer.activateSpan; +import static datadog.trace.bootstrap.instrumentation.api.AgentTracer.activeSpan; +import static datadog.trace.bootstrap.instrumentation.api.AgentTracer.propagate; +import static datadog.trace.bootstrap.instrumentation.api.AgentTracer.startSpan; +import static datadog.trace.instrumentation.play24.PlayHeaders.GETTER; +import static datadog.trace.instrumentation.play24.PlayHttpServerDecorator.DECORATE; + +import datadog.trace.bootstrap.instrumentation.api.AgentScope; +import datadog.trace.bootstrap.instrumentation.api.AgentSpan; +import datadog.trace.bootstrap.instrumentation.api.AgentSpan.Context; +import datadog.trace.bootstrap.instrumentation.api.Tags; +import net.bytebuddy.asm.Advice; +import play.api.mvc.Action; +import play.api.mvc.Request; +import play.api.mvc.Result; +import scala.concurrent.Future; + +public class PlayAdvice { + @Advice.OnMethodEnter(suppress = Throwable.class) + public static AgentScope onEnter(@Advice.Argument(0) final Request req) { + final AgentSpan span; + if (activeSpan() == null) { + final Context extractedContext = propagate().extract(req.headers(), GETTER); + span = startSpan("play.request", extractedContext); + } else { + // An upstream framework (e.g. akka-http, netty) has already started the span. + // Do not extract the context. + span = startSpan("play.request"); + } + DECORATE.afterStart(span); + DECORATE.onConnection(span, req); + + final AgentScope scope = activateSpan(span, false); + scope.setAsyncPropagation(true); + return scope; + } + + @Advice.OnMethodExit(onThrowable = Throwable.class, suppress = Throwable.class) + public static void stopTraceOnResponse( + @Advice.Enter final AgentScope playControllerScope, + @Advice.This final Object thisAction, + @Advice.Thrown final Throwable throwable, + @Advice.Argument(0) final Request req, + @Advice.Return(readOnly = false) final Future responseFuture) { + final AgentSpan playControllerSpan = playControllerScope.span(); + + // Call onRequest on return after tags are populated. + DECORATE.onRequest(playControllerSpan, req); + + if (throwable == null) { + responseFuture.onComplete( + new RequestCompleteCallback(playControllerSpan), + ((Action) thisAction).executionContext()); + } else { + DECORATE.onError(playControllerSpan, throwable); + playControllerSpan.setTag(Tags.HTTP_STATUS, 500); + DECORATE.beforeFinish(playControllerSpan); + playControllerSpan.finish(); + } + playControllerScope.close(); + + final AgentSpan rootSpan = activeSpan(); + // set the resource name on the upstream akka/netty span + DECORATE.onRequest(rootSpan, req); + } + + // Unused method for muzzle to allow only 2.4-2.5 + public static void muzzleCheck() { + play.libs.Akka.system(); + } +} diff --git a/dd-java-agent/instrumentation/play-2.3/src/main/java8/datadog/trace/instrumentation/play24/PlayHeaders.java b/dd-java-agent/instrumentation/play-2.3/src/main/java8/datadog/trace/instrumentation/play24/PlayHeaders.java new file mode 100644 index 0000000000..fd8c22ea83 --- /dev/null +++ b/dd-java-agent/instrumentation/play-2.3/src/main/java8/datadog/trace/instrumentation/play24/PlayHeaders.java @@ -0,0 +1,26 @@ +package datadog.trace.instrumentation.play24; + +import datadog.trace.bootstrap.instrumentation.api.AgentPropagation; +import play.api.mvc.Headers; +import scala.Option; +import scala.collection.JavaConversions; + +public class PlayHeaders implements AgentPropagation.Getter { + + public static final PlayHeaders GETTER = new PlayHeaders(); + + @Override + public Iterable keys(final Headers headers) { + return JavaConversions.asJavaIterable(headers.keys()); + } + + @Override + public String get(final Headers headers, final String key) { + final Option option = headers.get(key); + if (option.isDefined()) { + return option.get(); + } else { + return null; + } + } +} diff --git a/dd-java-agent/instrumentation/play-2.3/src/main/java8/datadog/trace/instrumentation/play24/PlayHttpServerDecorator.java b/dd-java-agent/instrumentation/play-2.3/src/main/java8/datadog/trace/instrumentation/play24/PlayHttpServerDecorator.java new file mode 100644 index 0000000000..6c12b2f301 --- /dev/null +++ b/dd-java-agent/instrumentation/play-2.3/src/main/java8/datadog/trace/instrumentation/play24/PlayHttpServerDecorator.java @@ -0,0 +1,86 @@ +package datadog.trace.instrumentation.play24; + +import datadog.trace.api.DDTags; +import datadog.trace.bootstrap.instrumentation.api.AgentSpan; +import datadog.trace.bootstrap.instrumentation.api.Tags; +import datadog.trace.bootstrap.instrumentation.decorator.HttpServerDecorator; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.UndeclaredThrowableException; +import java.net.URI; +import java.net.URISyntaxException; +import lombok.extern.slf4j.Slf4j; +import play.api.mvc.Request; +import play.api.mvc.Result; +import scala.Option; + +@Slf4j +public class PlayHttpServerDecorator extends HttpServerDecorator { + public static final PlayHttpServerDecorator DECORATE = new PlayHttpServerDecorator(); + + @Override + protected String[] instrumentationNames() { + return new String[] {"play"}; + } + + @Override + protected String component() { + return "play-action"; + } + + @Override + protected String method(final Request httpRequest) { + return httpRequest.method(); + } + + @Override + protected URI url(final Request request) throws URISyntaxException { + return new URI((request.secure() ? "https://" : "http://") + request.host() + request.uri()); + } + + @Override + protected String peerHostIP(final Request request) { + return request.remoteAddress(); + } + + @Override + protected Integer peerPort(final Request request) { + return null; + } + + @Override + protected Integer status(final Result httpResponse) { + return httpResponse.header().status(); + } + + @Override + public AgentSpan onRequest(final AgentSpan span, final Request request) { + super.onRequest(span, request); + if (request != null) { + // more about routes here: + // https://github.com/playframework/playframework/blob/master/documentation/manual/releases/release26/migration26/Migration26.md#router-tags-are-now-attributes + final Option pathOption = request.tags().get("ROUTE_PATTERN"); + if (!pathOption.isEmpty()) { + final String path = (String) pathOption.get(); + span.setTag(DDTags.RESOURCE_NAME, request.method() + " " + path); + } + } + return span; + } + + @Override + public AgentSpan onError(final AgentSpan span, Throwable throwable) { + span.setTag(Tags.HTTP_STATUS, 500); + if (throwable != null + // This can be moved to instanceof check when using Java 8. + && throwable.getClass().getName().equals("java.util.concurrent.CompletionException") + && throwable.getCause() != null) { + throwable = throwable.getCause(); + } + while ((throwable instanceof InvocationTargetException + || throwable instanceof UndeclaredThrowableException) + && throwable.getCause() != null) { + throwable = throwable.getCause(); + } + return super.onError(span, throwable); + } +} diff --git a/dd-java-agent/instrumentation/play-2.3/src/main/java8/datadog/trace/instrumentation/play24/RequestCompleteCallback.java b/dd-java-agent/instrumentation/play-2.3/src/main/java8/datadog/trace/instrumentation/play24/RequestCompleteCallback.java new file mode 100644 index 0000000000..cb04d49441 --- /dev/null +++ b/dd-java-agent/instrumentation/play-2.3/src/main/java8/datadog/trace/instrumentation/play24/RequestCompleteCallback.java @@ -0,0 +1,40 @@ +package datadog.trace.instrumentation.play24; + +import static datadog.trace.bootstrap.instrumentation.api.AgentTracer.activeScope; +import static datadog.trace.instrumentation.play24.PlayHttpServerDecorator.DECORATE; + +import datadog.trace.context.TraceScope; +import datadog.trace.bootstrap.instrumentation.api.AgentSpan; +import lombok.extern.slf4j.Slf4j; +import play.api.mvc.Result; +import scala.util.Try; + +@Slf4j +public class RequestCompleteCallback extends scala.runtime.AbstractFunction1, Object> { + private final AgentSpan span; + + public RequestCompleteCallback(final AgentSpan span) { + this.span = span; + } + + @Override + public Object apply(final Try result) { + try { + if (result.isFailure()) { + DECORATE.onError(span, result.failed().get()); + } else { + DECORATE.onResponse(span, result.get()); + } + DECORATE.beforeFinish(span); + final TraceScope scope = activeScope(); + if (scope != null) { + scope.setAsyncPropagation(false); + } + } catch (final Throwable t) { + log.debug("error in play instrumentation", t); + } finally { + span.finish(); + } + return null; + } +} diff --git a/dd-java-agent/instrumentation/play-2.3/src/test/groovy/client/PlayWSClientTest.groovy b/dd-java-agent/instrumentation/play-2.3/src/test/groovy/client/PlayWSClientTest.groovy new file mode 100644 index 0000000000..f408a5cdc6 --- /dev/null +++ b/dd-java-agent/instrumentation/play-2.3/src/test/groovy/client/PlayWSClientTest.groovy @@ -0,0 +1,54 @@ +package client + +import datadog.trace.agent.test.base.HttpClientTest +import play.libs.ws.WS +import spock.lang.AutoCleanup +import spock.lang.Shared +import spock.lang.Subject + +import java.util.concurrent.TimeUnit + +// Play 2.6+ uses a separately versioned client that shades the underlying dependency +// This means our built in instrumentation won't work. +class PlayWSClientTest extends HttpClientTest { + @Subject + @Shared + @AutoCleanup + def client = WS.client() + + @Override + int doRequest(String method, URI uri, Map headers, Closure callback) { + def request = client.url(uri.toString()) + headers.entrySet().each { + request.setHeader(it.key, it.value) + } + + def status = request.execute(method).map({ + callback?.call() + it + }).map({ + it.status + }) + return status.get(1, TimeUnit.SECONDS) + } + + @Override + String component() { + return "" // NettyHttpClientDecorator.DECORATE.component() + } + + @Override + String expectedOperationName() { + return "netty.client.request" + } + + @Override + boolean testRedirects() { + false + } + + @Override + boolean testConnectionFailure() { + false + } +} diff --git a/dd-java-agent/instrumentation/play-2.3/src/test/groovy/server/NettyServerTestInstrumentation.java b/dd-java-agent/instrumentation/play-2.3/src/test/groovy/server/NettyServerTestInstrumentation.java new file mode 100644 index 0000000000..61cb150a54 --- /dev/null +++ b/dd-java-agent/instrumentation/play-2.3/src/test/groovy/server/NettyServerTestInstrumentation.java @@ -0,0 +1,22 @@ +package server; + +import static net.bytebuddy.matcher.ElementMatchers.named; + +import com.google.auto.service.AutoService; +import datadog.trace.agent.test.base.HttpServerTestAdvice; +import datadog.trace.agent.tooling.Instrumenter; +import net.bytebuddy.agent.builder.AgentBuilder; + +@AutoService(Instrumenter.class) +public class NettyServerTestInstrumentation implements Instrumenter { + + @Override + public AgentBuilder instrument(final AgentBuilder agentBuilder) { + return agentBuilder + .type(named("io.netty.handler.codec.ByteToMessageDecoder")) + .transform( + new AgentBuilder.Transformer.ForAdvice() + .advice( + named("channelRead"), HttpServerTestAdvice.ServerEntryAdvice.class.getName())); + } +} diff --git a/dd-java-agent/instrumentation/play-2.3/src/test/groovy/server/PlayAsyncServerTest.groovy b/dd-java-agent/instrumentation/play-2.3/src/test/groovy/server/PlayAsyncServerTest.groovy new file mode 100644 index 0000000000..38953b437d --- /dev/null +++ b/dd-java-agent/instrumentation/play-2.3/src/test/groovy/server/PlayAsyncServerTest.groovy @@ -0,0 +1,46 @@ +package server + +class PlayAsyncServerTest { //extends PlayServerTest { +// @Override +// Server startServer(int port) { +// def router = +// new RoutingDsl() +// .GET(SUCCESS.getPath()).routeAsync({ +// CompletableFuture.supplyAsync({ +// controller(SUCCESS) { +// Results.status(SUCCESS.getStatus(), SUCCESS.getBody()) +// } +// }, HttpExecution.defaultContext()) +// } as Supplier) +// .GET(QUERY_PARAM.getPath()).routeAsync({ +// CompletableFuture.supplyAsync({ +// controller(QUERY_PARAM) { +// Results.status(QUERY_PARAM.getStatus(), QUERY_PARAM.getBody()) +// } +// }, HttpExecution.defaultContext()) +// } as Supplier) +// .GET(REDIRECT.getPath()).routeAsync({ +// CompletableFuture.supplyAsync({ +// controller(REDIRECT) { +// Results.found(REDIRECT.getBody()) +// } +// }, HttpExecution.defaultContext()) +// } as Supplier) +// .GET(ERROR.getPath()).routeAsync({ +// CompletableFuture.supplyAsync({ +// controller(ERROR) { +// Results.status(ERROR.getStatus(), ERROR.getBody()) +// } +// }, HttpExecution.defaultContext()) +// } as Supplier) +// .GET(EXCEPTION.getPath()).routeAsync({ +// CompletableFuture.supplyAsync({ +// controller(EXCEPTION) { +// throw new Exception(EXCEPTION.getBody()) +// } +// }, HttpExecution.defaultContext()) +// } as Supplier) +// +// return Server.forRouter(router.build(), port) +// } +} diff --git a/dd-java-agent/instrumentation/play-2.3/src/test/groovy/server/PlayServerTest.groovy b/dd-java-agent/instrumentation/play-2.3/src/test/groovy/server/PlayServerTest.groovy new file mode 100644 index 0000000000..95fa6a0027 --- /dev/null +++ b/dd-java-agent/instrumentation/play-2.3/src/test/groovy/server/PlayServerTest.groovy @@ -0,0 +1,79 @@ +package server + +import datadog.opentracing.DDSpan +import datadog.trace.agent.test.asserts.TraceAssert +import datadog.trace.agent.test.base.HttpServerTest +import datadog.trace.api.DDSpanTypes +import datadog.trace.api.DDTags +import datadog.trace.bootstrap.instrumentation.api.Tags +import play.api.test.TestServer + +import static datadog.trace.agent.test.base.HttpServerTest.ServerEndpoint.* + +class PlayServerTest extends HttpServerTest { + @Override + TestServer startServer(int port) { + def server = SyncServer.server(port) + server.start() + return server + } + + @Override + void stopServer(TestServer server) { + server.stop() + } + + @Override + String component() { + return "" + } + + @Override + String expectedOperationName() { + return "netty.request" + } + + @Override + boolean hasHandlerSpan() { + true + } + + @Override + // Return the handler span's name + String reorderHandlerSpan() { + "play.request" + } + + boolean testExceptionBody() { + // I can't figure out how to set a proper exception handler to customize the response body. + false + } + + @Override + void handlerSpan(TraceAssert trace, int index, Object parent, ServerEndpoint endpoint = SUCCESS) { + trace.span(index) { + serviceName expectedServiceName() + operationName "play.request" + spanType DDSpanTypes.HTTP_SERVER + errored endpoint == ERROR || endpoint == EXCEPTION + childOf(parent as DDSpan) + tags { + "$Tags.COMPONENT" "" + "$Tags.SPAN_KIND" Tags.SPAN_KIND_SERVER + "$Tags.PEER_HOST_IPV4" { it == null || it == "127.0.0.1" } // Optional + "$Tags.HTTP_URL" String + "$Tags.HTTP_METHOD" String + "$Tags.HTTP_STATUS" Integer + if (endpoint == ERROR) { + "$Tags.ERROR" true + } else if (endpoint == EXCEPTION) { + errorTags(Exception, EXCEPTION.body) + } + if (endpoint.query) { + "$DDTags.HTTP_QUERY" endpoint.query + } + defaultTags() + } + } + } +} diff --git a/dd-java-agent/instrumentation/play-2.3/src/test/scala/server/AsyncServer.scala b/dd-java-agent/instrumentation/play-2.3/src/test/scala/server/AsyncServer.scala new file mode 100644 index 0000000000..228dcb95a3 --- /dev/null +++ b/dd-java-agent/instrumentation/play-2.3/src/test/scala/server/AsyncServer.scala @@ -0,0 +1,5 @@ +package server + +object AsyncServer { + +} diff --git a/dd-java-agent/instrumentation/play-2.3/src/test/scala/server/SyncServer.scala b/dd-java-agent/instrumentation/play-2.3/src/test/scala/server/SyncServer.scala new file mode 100644 index 0000000000..1f36d66824 --- /dev/null +++ b/dd-java-agent/instrumentation/play-2.3/src/test/scala/server/SyncServer.scala @@ -0,0 +1,15 @@ +package server + +import datadog.trace.agent.test.base.HttpServerTest.ServerEndpoint._ +import play.api.mvc.{Action, Handler, Results} +import play.api.test.{FakeApplication, TestServer} + +object SyncServer { + val routes: PartialFunction[(String, String), Handler] = { + case ("GET", "/success") => Action { request => Results.Status(SUCCESS.getStatus) } + } + + def server(port: Int): TestServer = { + TestServer(port, FakeApplication(withRoutes = routes)) + } +} diff --git a/settings.gradle b/settings.gradle index e184365078..9e75b55c12 100644 --- a/settings.gradle +++ b/settings.gradle @@ -116,6 +116,7 @@ include ':dd-java-agent:instrumentation:netty-3.8' include ':dd-java-agent:instrumentation:netty-4.0' include ':dd-java-agent:instrumentation:netty-4.1' include ':dd-java-agent:instrumentation:okhttp-3' +include ':dd-java-agent:instrumentation:play-2.3' include ':dd-java-agent:instrumentation:play-2.4' include ':dd-java-agent:instrumentation:play-2.6' include ':dd-java-agent:instrumentation:play-ws' From 5cfaf8b2e57403e40399d450f9363872c38727ea Mon Sep 17 00:00:00 2001 From: Brian Devins-Suresh Date: Tue, 17 Mar 2020 16:49:19 -0400 Subject: [PATCH 14/43] Test servers now both work --- .../groovy/client/PlayWSClientTest.groovy | 2 - .../groovy/server/PlayAsyncServerTest.groovy | 52 ++++--------------- .../test/groovy/server/PlayServerTest.groovy | 5 -- .../src/test/scala/server/AsyncServer.scala | 25 ++++++++- .../src/test/scala/server/Settings.scala | 12 +++++ .../src/test/scala/server/SyncServer.scala | 8 ++- 6 files changed, 51 insertions(+), 53 deletions(-) create mode 100644 dd-java-agent/instrumentation/play-2.3/src/test/scala/server/Settings.scala diff --git a/dd-java-agent/instrumentation/play-2.3/src/test/groovy/client/PlayWSClientTest.groovy b/dd-java-agent/instrumentation/play-2.3/src/test/groovy/client/PlayWSClientTest.groovy index f408a5cdc6..2e0b8bfce9 100644 --- a/dd-java-agent/instrumentation/play-2.3/src/test/groovy/client/PlayWSClientTest.groovy +++ b/dd-java-agent/instrumentation/play-2.3/src/test/groovy/client/PlayWSClientTest.groovy @@ -8,8 +8,6 @@ import spock.lang.Subject import java.util.concurrent.TimeUnit -// Play 2.6+ uses a separately versioned client that shades the underlying dependency -// This means our built in instrumentation won't work. class PlayWSClientTest extends HttpClientTest { @Subject @Shared diff --git a/dd-java-agent/instrumentation/play-2.3/src/test/groovy/server/PlayAsyncServerTest.groovy b/dd-java-agent/instrumentation/play-2.3/src/test/groovy/server/PlayAsyncServerTest.groovy index 38953b437d..71dfc6a9c6 100644 --- a/dd-java-agent/instrumentation/play-2.3/src/test/groovy/server/PlayAsyncServerTest.groovy +++ b/dd-java-agent/instrumentation/play-2.3/src/test/groovy/server/PlayAsyncServerTest.groovy @@ -1,46 +1,12 @@ package server -class PlayAsyncServerTest { //extends PlayServerTest { -// @Override -// Server startServer(int port) { -// def router = -// new RoutingDsl() -// .GET(SUCCESS.getPath()).routeAsync({ -// CompletableFuture.supplyAsync({ -// controller(SUCCESS) { -// Results.status(SUCCESS.getStatus(), SUCCESS.getBody()) -// } -// }, HttpExecution.defaultContext()) -// } as Supplier) -// .GET(QUERY_PARAM.getPath()).routeAsync({ -// CompletableFuture.supplyAsync({ -// controller(QUERY_PARAM) { -// Results.status(QUERY_PARAM.getStatus(), QUERY_PARAM.getBody()) -// } -// }, HttpExecution.defaultContext()) -// } as Supplier) -// .GET(REDIRECT.getPath()).routeAsync({ -// CompletableFuture.supplyAsync({ -// controller(REDIRECT) { -// Results.found(REDIRECT.getBody()) -// } -// }, HttpExecution.defaultContext()) -// } as Supplier) -// .GET(ERROR.getPath()).routeAsync({ -// CompletableFuture.supplyAsync({ -// controller(ERROR) { -// Results.status(ERROR.getStatus(), ERROR.getBody()) -// } -// }, HttpExecution.defaultContext()) -// } as Supplier) -// .GET(EXCEPTION.getPath()).routeAsync({ -// CompletableFuture.supplyAsync({ -// controller(EXCEPTION) { -// throw new Exception(EXCEPTION.getBody()) -// } -// }, HttpExecution.defaultContext()) -// } as Supplier) -// -// return Server.forRouter(router.build(), port) -// } +import play.api.test.TestServer + +class PlayAsyncServerTest extends PlayServerTest { + @Override + TestServer startServer(int port) { + def server = AsyncServer.server(port) + server.start() + return server + } } diff --git a/dd-java-agent/instrumentation/play-2.3/src/test/groovy/server/PlayServerTest.groovy b/dd-java-agent/instrumentation/play-2.3/src/test/groovy/server/PlayServerTest.groovy index 95fa6a0027..f6a733e58d 100644 --- a/dd-java-agent/instrumentation/play-2.3/src/test/groovy/server/PlayServerTest.groovy +++ b/dd-java-agent/instrumentation/play-2.3/src/test/groovy/server/PlayServerTest.groovy @@ -44,11 +44,6 @@ class PlayServerTest extends HttpServerTest { "play.request" } - boolean testExceptionBody() { - // I can't figure out how to set a proper exception handler to customize the response body. - false - } - @Override void handlerSpan(TraceAssert trace, int index, Object parent, ServerEndpoint endpoint = SUCCESS) { trace.span(index) { diff --git a/dd-java-agent/instrumentation/play-2.3/src/test/scala/server/AsyncServer.scala b/dd-java-agent/instrumentation/play-2.3/src/test/scala/server/AsyncServer.scala index 228dcb95a3..087f520a85 100644 --- a/dd-java-agent/instrumentation/play-2.3/src/test/scala/server/AsyncServer.scala +++ b/dd-java-agent/instrumentation/play-2.3/src/test/scala/server/AsyncServer.scala @@ -1,5 +1,28 @@ package server -object AsyncServer { +import datadog.trace.agent.test.base.HttpServerTest.ServerEndpoint._ +import play.api.libs.concurrent.Execution.{defaultContext => ec} +import play.api.mvc.{Action, Handler, Results} +import play.api.test.{FakeApplication, TestServer} +import scala.concurrent.Future + +object AsyncServer { + val routes: PartialFunction[(String, String), Handler] = { + case ("GET", "/success") => Action.async { request => Future.successful(Results.Status(SUCCESS.getStatus).apply(SUCCESS.getBody)) } + case ("GET", "/redirect") => Action.async { request => Future.successful(Results.Redirect(REDIRECT.getBody, REDIRECT.getStatus)) } + case ("GET", "/query") => Action.async { result => Future.successful(Results.Status(QUERY_PARAM.getStatus).apply(QUERY_PARAM.getBody)) } + case ("GET", "/error-status") => Action.async { result => Future { + throw new RuntimeException(ERROR.getBody) + }(ec) + } + case ("GET", "/exception") => Action.async { result => Future { + throw new RuntimeException(ERROR.getBody) + }(ec) + } + } + + def server(port: Int): TestServer = { + TestServer(port, FakeApplication(withGlobal = Some(new Settings()), withRoutes = routes)) + } } diff --git a/dd-java-agent/instrumentation/play-2.3/src/test/scala/server/Settings.scala b/dd-java-agent/instrumentation/play-2.3/src/test/scala/server/Settings.scala new file mode 100644 index 0000000000..04add7c804 --- /dev/null +++ b/dd-java-agent/instrumentation/play-2.3/src/test/scala/server/Settings.scala @@ -0,0 +1,12 @@ +package server + +import play.api.GlobalSettings +import play.api.mvc.{RequestHeader, Result, Results} + +import scala.concurrent.Future + +class Settings extends GlobalSettings { + override def onError(request: RequestHeader, ex: Throwable): Future[Result] = { + Future.successful(Results.InternalServerError(ex.getCause.getMessage)) + } +} diff --git a/dd-java-agent/instrumentation/play-2.3/src/test/scala/server/SyncServer.scala b/dd-java-agent/instrumentation/play-2.3/src/test/scala/server/SyncServer.scala index 1f36d66824..699a900164 100644 --- a/dd-java-agent/instrumentation/play-2.3/src/test/scala/server/SyncServer.scala +++ b/dd-java-agent/instrumentation/play-2.3/src/test/scala/server/SyncServer.scala @@ -6,10 +6,14 @@ import play.api.test.{FakeApplication, TestServer} object SyncServer { val routes: PartialFunction[(String, String), Handler] = { - case ("GET", "/success") => Action { request => Results.Status(SUCCESS.getStatus) } + case ("GET", "/success") => Action { request => Results.Status(SUCCESS.getStatus).apply(SUCCESS.getBody) } + case ("GET", "/redirect") => Action { request => Results.Redirect(REDIRECT.getBody, REDIRECT.getStatus) } + case ("GET", "/query") => Action { result => Results.Status(QUERY_PARAM.getStatus).apply(QUERY_PARAM.getBody) } + case ("GET", "/error-status") => Action { result => throw new RuntimeException(ERROR.getBody) } + case ("GET", "/exception") => Action { result => throw new RuntimeException(EXCEPTION.getBody) } } def server(port: Int): TestServer = { - TestServer(port, FakeApplication(withRoutes = routes)) + TestServer(port, FakeApplication(withGlobal = Some(new Settings()), withRoutes = routes)) } } From d1093a79b8e5d412dcdf51c4116874777dc39e1a Mon Sep 17 00:00:00 2001 From: Brian Devins-Suresh Date: Tue, 17 Mar 2020 16:52:50 -0400 Subject: [PATCH 15/43] rename package --- dd-java-agent/instrumentation/play-2.3/play-2.3.gradle | 2 +- .../{play24 => play23}/PlayInstrumentation.java | 2 +- .../instrumentation/{play24 => play23}/PlayAdvice.java | 7 +++---- .../instrumentation/{play24 => play23}/PlayHeaders.java | 2 +- .../{play24 => play23}/PlayHttpServerDecorator.java | 2 +- .../{play24 => play23}/RequestCompleteCallback.java | 6 +++--- 6 files changed, 10 insertions(+), 11 deletions(-) rename dd-java-agent/instrumentation/play-2.3/src/main/java/datadog/trace/instrumentation/{play24 => play23}/PlayInstrumentation.java (97%) rename dd-java-agent/instrumentation/play-2.3/src/main/java8/datadog/trace/instrumentation/{play24 => play23}/PlayAdvice.java (93%) rename dd-java-agent/instrumentation/play-2.3/src/main/java8/datadog/trace/instrumentation/{play24 => play23}/PlayHeaders.java (93%) rename dd-java-agent/instrumentation/play-2.3/src/main/java8/datadog/trace/instrumentation/{play24 => play23}/PlayHttpServerDecorator.java (98%) rename dd-java-agent/instrumentation/play-2.3/src/main/java8/datadog/trace/instrumentation/{play24 => play23}/RequestCompleteCallback.java (90%) diff --git a/dd-java-agent/instrumentation/play-2.3/play-2.3.gradle b/dd-java-agent/instrumentation/play-2.3/play-2.3.gradle index e77bad0dea..cbe2b9a76e 100644 --- a/dd-java-agent/instrumentation/play-2.3/play-2.3.gradle +++ b/dd-java-agent/instrumentation/play-2.3/play-2.3.gradle @@ -8,7 +8,7 @@ muzzle { pass { group = 'com.typesafe.play' module = 'play_2.11' - versions = '[2.3.0,2.4)' + versions = '[2.3.9,2.4)' assertInverse = true } fail { diff --git a/dd-java-agent/instrumentation/play-2.3/src/main/java/datadog/trace/instrumentation/play24/PlayInstrumentation.java b/dd-java-agent/instrumentation/play-2.3/src/main/java/datadog/trace/instrumentation/play23/PlayInstrumentation.java similarity index 97% rename from dd-java-agent/instrumentation/play-2.3/src/main/java/datadog/trace/instrumentation/play24/PlayInstrumentation.java rename to dd-java-agent/instrumentation/play-2.3/src/main/java/datadog/trace/instrumentation/play23/PlayInstrumentation.java index fa7d265158..a8ac6a72d5 100644 --- a/dd-java-agent/instrumentation/play-2.3/src/main/java/datadog/trace/instrumentation/play24/PlayInstrumentation.java +++ b/dd-java-agent/instrumentation/play-2.3/src/main/java/datadog/trace/instrumentation/play23/PlayInstrumentation.java @@ -1,4 +1,4 @@ -package datadog.trace.instrumentation.play24; +package datadog.trace.instrumentation.play23; import static datadog.trace.agent.tooling.ClassLoaderMatcher.hasClassesNamed; import static datadog.trace.agent.tooling.bytebuddy.matcher.DDElementMatchers.implementsInterface; diff --git a/dd-java-agent/instrumentation/play-2.3/src/main/java8/datadog/trace/instrumentation/play24/PlayAdvice.java b/dd-java-agent/instrumentation/play-2.3/src/main/java8/datadog/trace/instrumentation/play23/PlayAdvice.java similarity index 93% rename from dd-java-agent/instrumentation/play-2.3/src/main/java8/datadog/trace/instrumentation/play24/PlayAdvice.java rename to dd-java-agent/instrumentation/play-2.3/src/main/java8/datadog/trace/instrumentation/play23/PlayAdvice.java index 2e495f2e72..924dbde45f 100644 --- a/dd-java-agent/instrumentation/play-2.3/src/main/java8/datadog/trace/instrumentation/play24/PlayAdvice.java +++ b/dd-java-agent/instrumentation/play-2.3/src/main/java8/datadog/trace/instrumentation/play23/PlayAdvice.java @@ -1,11 +1,10 @@ -package datadog.trace.instrumentation.play24; +package datadog.trace.instrumentation.play23; import static datadog.trace.bootstrap.instrumentation.api.AgentTracer.activateSpan; import static datadog.trace.bootstrap.instrumentation.api.AgentTracer.activeSpan; import static datadog.trace.bootstrap.instrumentation.api.AgentTracer.propagate; import static datadog.trace.bootstrap.instrumentation.api.AgentTracer.startSpan; -import static datadog.trace.instrumentation.play24.PlayHeaders.GETTER; -import static datadog.trace.instrumentation.play24.PlayHttpServerDecorator.DECORATE; +import static datadog.trace.instrumentation.play23.PlayHttpServerDecorator.DECORATE; import datadog.trace.bootstrap.instrumentation.api.AgentScope; import datadog.trace.bootstrap.instrumentation.api.AgentSpan; @@ -22,7 +21,7 @@ public class PlayAdvice { public static AgentScope onEnter(@Advice.Argument(0) final Request req) { final AgentSpan span; if (activeSpan() == null) { - final Context extractedContext = propagate().extract(req.headers(), GETTER); + final Context extractedContext = propagate().extract(req.headers(), PlayHeaders.GETTER); span = startSpan("play.request", extractedContext); } else { // An upstream framework (e.g. akka-http, netty) has already started the span. diff --git a/dd-java-agent/instrumentation/play-2.3/src/main/java8/datadog/trace/instrumentation/play24/PlayHeaders.java b/dd-java-agent/instrumentation/play-2.3/src/main/java8/datadog/trace/instrumentation/play23/PlayHeaders.java similarity index 93% rename from dd-java-agent/instrumentation/play-2.3/src/main/java8/datadog/trace/instrumentation/play24/PlayHeaders.java rename to dd-java-agent/instrumentation/play-2.3/src/main/java8/datadog/trace/instrumentation/play23/PlayHeaders.java index fd8c22ea83..a0c33050be 100644 --- a/dd-java-agent/instrumentation/play-2.3/src/main/java8/datadog/trace/instrumentation/play24/PlayHeaders.java +++ b/dd-java-agent/instrumentation/play-2.3/src/main/java8/datadog/trace/instrumentation/play23/PlayHeaders.java @@ -1,4 +1,4 @@ -package datadog.trace.instrumentation.play24; +package datadog.trace.instrumentation.play23; import datadog.trace.bootstrap.instrumentation.api.AgentPropagation; import play.api.mvc.Headers; diff --git a/dd-java-agent/instrumentation/play-2.3/src/main/java8/datadog/trace/instrumentation/play24/PlayHttpServerDecorator.java b/dd-java-agent/instrumentation/play-2.3/src/main/java8/datadog/trace/instrumentation/play23/PlayHttpServerDecorator.java similarity index 98% rename from dd-java-agent/instrumentation/play-2.3/src/main/java8/datadog/trace/instrumentation/play24/PlayHttpServerDecorator.java rename to dd-java-agent/instrumentation/play-2.3/src/main/java8/datadog/trace/instrumentation/play23/PlayHttpServerDecorator.java index 6c12b2f301..09878f9ba0 100644 --- a/dd-java-agent/instrumentation/play-2.3/src/main/java8/datadog/trace/instrumentation/play24/PlayHttpServerDecorator.java +++ b/dd-java-agent/instrumentation/play-2.3/src/main/java8/datadog/trace/instrumentation/play23/PlayHttpServerDecorator.java @@ -1,4 +1,4 @@ -package datadog.trace.instrumentation.play24; +package datadog.trace.instrumentation.play23; import datadog.trace.api.DDTags; import datadog.trace.bootstrap.instrumentation.api.AgentSpan; diff --git a/dd-java-agent/instrumentation/play-2.3/src/main/java8/datadog/trace/instrumentation/play24/RequestCompleteCallback.java b/dd-java-agent/instrumentation/play-2.3/src/main/java8/datadog/trace/instrumentation/play23/RequestCompleteCallback.java similarity index 90% rename from dd-java-agent/instrumentation/play-2.3/src/main/java8/datadog/trace/instrumentation/play24/RequestCompleteCallback.java rename to dd-java-agent/instrumentation/play-2.3/src/main/java8/datadog/trace/instrumentation/play23/RequestCompleteCallback.java index cb04d49441..e1c5ff9dc3 100644 --- a/dd-java-agent/instrumentation/play-2.3/src/main/java8/datadog/trace/instrumentation/play24/RequestCompleteCallback.java +++ b/dd-java-agent/instrumentation/play-2.3/src/main/java8/datadog/trace/instrumentation/play23/RequestCompleteCallback.java @@ -1,10 +1,10 @@ -package datadog.trace.instrumentation.play24; +package datadog.trace.instrumentation.play23; import static datadog.trace.bootstrap.instrumentation.api.AgentTracer.activeScope; -import static datadog.trace.instrumentation.play24.PlayHttpServerDecorator.DECORATE; +import static datadog.trace.instrumentation.play23.PlayHttpServerDecorator.DECORATE; -import datadog.trace.context.TraceScope; import datadog.trace.bootstrap.instrumentation.api.AgentSpan; +import datadog.trace.context.TraceScope; import lombok.extern.slf4j.Slf4j; import play.api.mvc.Result; import scala.util.Try; From 89a0761ab4b0660d24b77abe8f5aaab640503c31 Mon Sep 17 00:00:00 2001 From: Brian Devins-Suresh Date: Wed, 18 Mar 2020 10:26:55 -0400 Subject: [PATCH 16/43] Add typesafe repo and skip bad builds --- buildSrc/src/main/groovy/MuzzlePlugin.groovy | 15 ++++++++------- .../instrumentation/play-2.3/play-2.3.gradle | 10 +++++----- gradle/java.gradle | 3 +++ 3 files changed, 16 insertions(+), 12 deletions(-) diff --git a/buildSrc/src/main/groovy/MuzzlePlugin.groovy b/buildSrc/src/main/groovy/MuzzlePlugin.groovy index f8221de0d4..d8180344d8 100644 --- a/buildSrc/src/main/groovy/MuzzlePlugin.groovy +++ b/buildSrc/src/main/groovy/MuzzlePlugin.groovy @@ -14,16 +14,13 @@ import org.eclipse.aether.spi.connector.RepositoryConnectorFactory import org.eclipse.aether.spi.connector.transport.TransporterFactory import org.eclipse.aether.transport.http.HttpTransporterFactory import org.eclipse.aether.version.Version -import org.gradle.api.Action -import org.gradle.api.GradleException -import org.gradle.api.Plugin -import org.gradle.api.Project -import org.gradle.api.Task +import org.gradle.api.* import org.gradle.api.model.ObjectFactory import java.lang.reflect.Method import java.security.SecureClassLoader import java.util.concurrent.atomic.AtomicReference +import java.util.regex.Pattern /** * muzzle task plugin which runs muzzle validation against a range of dependencies. @@ -36,7 +33,8 @@ class MuzzlePlugin implements Plugin { private static final AtomicReference TOOLING_LOADER = new AtomicReference<>() static { RemoteRepository central = new RemoteRepository.Builder("central", "default", "https://repo1.maven.org/maven2/").build() - MUZZLE_REPOS = new ArrayList(Arrays.asList(central)) + RemoteRepository typesafe = new RemoteRepository.Builder("typesafe", "default", "https://repo.typesafe.com/typesafe/releases").build() + MUZZLE_REPOS = new ArrayList(Arrays.asList(central, typesafe)) } @Override @@ -343,6 +341,8 @@ class MuzzlePlugin implements Plugin { return session } + private static final Pattern GIT_SHA_PATTERN = Pattern.compile('^.*-[0-9a-f]{7,}$') + /** * Filter out snapshot-type builds from versions list. */ @@ -357,7 +357,8 @@ class MuzzlePlugin implements Plugin { version.contains(".m") || version.contains("-m") || version.contains("-dev") || - version.contains("public_draft") + version.contains("public_draft") || + version.matches(GIT_SHA_PATTERN) } return list } diff --git a/dd-java-agent/instrumentation/play-2.3/play-2.3.gradle b/dd-java-agent/instrumentation/play-2.3/play-2.3.gradle index cbe2b9a76e..7614f1d01b 100644 --- a/dd-java-agent/instrumentation/play-2.3/play-2.3.gradle +++ b/dd-java-agent/instrumentation/play-2.3/play-2.3.gradle @@ -8,7 +8,7 @@ muzzle { pass { group = 'com.typesafe.play' module = 'play_2.11' - versions = '[2.3.9,2.4)' + versions = '[2.3.0,2.4)' assertInverse = true } fail { @@ -35,11 +35,11 @@ testSets { } dependencies { - main_java8Compile group: 'com.typesafe.play', name: 'play_2.11', version: '2.3.9' + main_java8Compile group: 'com.typesafe.play', name: 'play_2.11', version: '2.3.0' - testCompile group: 'com.typesafe.play', name: 'play-java_2.11', version: '2.3.9' - testCompile group: 'com.typesafe.play', name: 'play-java-ws_2.11', version: '2.3.9' - testCompile(group: 'com.typesafe.play', name: 'play-test_2.11', version: '2.3.9') + testCompile group: 'com.typesafe.play', name: 'play-java_2.11', version: '2.3.0' + testCompile group: 'com.typesafe.play', name: 'play-java-ws_2.11', version: '2.3.0' + testCompile(group: 'com.typesafe.play', name: 'play-test_2.11', version: '2.3.0') latestDepTestCompile group: 'com.typesafe.play', name: 'play-java_2.11', version: '2.3.+' latestDepTestCompile group: 'com.typesafe.play', name: 'play-java-ws_2.11', version: '2.3.+' diff --git a/gradle/java.gradle b/gradle/java.gradle index d2ab140903..5f3e53061f 100644 --- a/gradle/java.gradle +++ b/gradle/java.gradle @@ -131,6 +131,9 @@ repositories { maven { url "https://adoptopenjdk.jfrog.io/adoptopenjdk/jmc-libs-snapshots" } + maven { + url "https://repo.typesafe.com/typesafe/releases" + } } dependencies { From 485d9cb6c26874663e93171c28bcb72ce182a56b Mon Sep 17 00:00:00 2001 From: Brian Devins-Suresh Date: Thu, 19 Mar 2020 16:49:38 -0400 Subject: [PATCH 17/43] Work so far --- .../play23/PlayInstrumentation.java | 2 +- .../instrumentation/play23/PlayAdvice.java | 5 ---- .../NettyServerTestInstrumentation.java | 5 ++-- .../test/groovy/server/PlayServerTest.groovy | 8 +++--- .../server/ControllerClosureAdapter.scala | 18 +++++++++++++ .../src/test/scala/server/SyncServer.scala | 25 +++++++++++++++---- .../agent/test/base/HttpServerTest.groovy | 9 ++----- .../trace/common/writer/ListWriter.java | 7 ++++++ 8 files changed, 56 insertions(+), 23 deletions(-) create mode 100644 dd-java-agent/instrumentation/play-2.3/src/test/scala/server/ControllerClosureAdapter.scala diff --git a/dd-java-agent/instrumentation/play-2.3/src/main/java/datadog/trace/instrumentation/play23/PlayInstrumentation.java b/dd-java-agent/instrumentation/play-2.3/src/main/java/datadog/trace/instrumentation/play23/PlayInstrumentation.java index a8ac6a72d5..1a7ca983e9 100644 --- a/dd-java-agent/instrumentation/play-2.3/src/main/java/datadog/trace/instrumentation/play23/PlayInstrumentation.java +++ b/dd-java-agent/instrumentation/play-2.3/src/main/java/datadog/trace/instrumentation/play23/PlayInstrumentation.java @@ -18,7 +18,7 @@ import net.bytebuddy.matcher.ElementMatcher; public final class PlayInstrumentation extends Instrumenter.Default { public PlayInstrumentation() { - super("play"); + super("play", "play-action"); } @Override diff --git a/dd-java-agent/instrumentation/play-2.3/src/main/java8/datadog/trace/instrumentation/play23/PlayAdvice.java b/dd-java-agent/instrumentation/play-2.3/src/main/java8/datadog/trace/instrumentation/play23/PlayAdvice.java index 924dbde45f..5c1de38c6b 100644 --- a/dd-java-agent/instrumentation/play-2.3/src/main/java8/datadog/trace/instrumentation/play23/PlayAdvice.java +++ b/dd-java-agent/instrumentation/play-2.3/src/main/java8/datadog/trace/instrumentation/play23/PlayAdvice.java @@ -64,9 +64,4 @@ public class PlayAdvice { // set the resource name on the upstream akka/netty span DECORATE.onRequest(rootSpan, req); } - - // Unused method for muzzle to allow only 2.4-2.5 - public static void muzzleCheck() { - play.libs.Akka.system(); - } } diff --git a/dd-java-agent/instrumentation/play-2.3/src/test/groovy/server/NettyServerTestInstrumentation.java b/dd-java-agent/instrumentation/play-2.3/src/test/groovy/server/NettyServerTestInstrumentation.java index 61cb150a54..b94ea96264 100644 --- a/dd-java-agent/instrumentation/play-2.3/src/test/groovy/server/NettyServerTestInstrumentation.java +++ b/dd-java-agent/instrumentation/play-2.3/src/test/groovy/server/NettyServerTestInstrumentation.java @@ -13,10 +13,11 @@ public class NettyServerTestInstrumentation implements Instrumenter { @Override public AgentBuilder instrument(final AgentBuilder agentBuilder) { return agentBuilder - .type(named("io.netty.handler.codec.ByteToMessageDecoder")) + .type(named("org.jboss.netty.handler.codec.http.HttpRequestDecoder")) .transform( new AgentBuilder.Transformer.ForAdvice() .advice( - named("channelRead"), HttpServerTestAdvice.ServerEntryAdvice.class.getName())); + named("createMessage"), + HttpServerTestAdvice.ServerEntryAdvice.class.getName())); } } diff --git a/dd-java-agent/instrumentation/play-2.3/src/test/groovy/server/PlayServerTest.groovy b/dd-java-agent/instrumentation/play-2.3/src/test/groovy/server/PlayServerTest.groovy index f6a733e58d..32aa2e83c9 100644 --- a/dd-java-agent/instrumentation/play-2.3/src/test/groovy/server/PlayServerTest.groovy +++ b/dd-java-agent/instrumentation/play-2.3/src/test/groovy/server/PlayServerTest.groovy @@ -6,6 +6,7 @@ import datadog.trace.agent.test.base.HttpServerTest import datadog.trace.api.DDSpanTypes import datadog.trace.api.DDTags import datadog.trace.bootstrap.instrumentation.api.Tags +import datadog.trace.instrumentation.play23.PlayHttpServerDecorator import play.api.test.TestServer import static datadog.trace.agent.test.base.HttpServerTest.ServerEndpoint.* @@ -25,14 +26,15 @@ class PlayServerTest extends HttpServerTest { @Override String component() { - return "" + return "netty" } @Override String expectedOperationName() { - return "netty.request" + return "play.server" } + // We don't have instrumentation for this version of netty yet @Override boolean hasHandlerSpan() { true @@ -53,7 +55,7 @@ class PlayServerTest extends HttpServerTest { errored endpoint == ERROR || endpoint == EXCEPTION childOf(parent as DDSpan) tags { - "$Tags.COMPONENT" "" + "$Tags.COMPONENT" PlayHttpServerDecorator.DECORATE.component() "$Tags.SPAN_KIND" Tags.SPAN_KIND_SERVER "$Tags.PEER_HOST_IPV4" { it == null || it == "127.0.0.1" } // Optional "$Tags.HTTP_URL" String diff --git a/dd-java-agent/instrumentation/play-2.3/src/test/scala/server/ControllerClosureAdapter.scala b/dd-java-agent/instrumentation/play-2.3/src/test/scala/server/ControllerClosureAdapter.scala new file mode 100644 index 0000000000..bf800b1c87 --- /dev/null +++ b/dd-java-agent/instrumentation/play-2.3/src/test/scala/server/ControllerClosureAdapter.scala @@ -0,0 +1,18 @@ +package server + +import groovy.lang.Closure +import play.api.mvc.Result + +import scala.concurrent.Future + +class ControllerClosureAdapter(response: Result) extends Closure[Result] { + override def call(): Result = response +} + +class BlockClosureAdapter(block: () => Result) extends Closure[Result] { + override def call(): Result = block() +} + +class AsyncControllerClosureAdapter(response: Future[Result]) extends Closure[Future[Result]] { + override def call(): Future[Result] = response +} diff --git a/dd-java-agent/instrumentation/play-2.3/src/test/scala/server/SyncServer.scala b/dd-java-agent/instrumentation/play-2.3/src/test/scala/server/SyncServer.scala index 699a900164..fad6181a13 100644 --- a/dd-java-agent/instrumentation/play-2.3/src/test/scala/server/SyncServer.scala +++ b/dd-java-agent/instrumentation/play-2.3/src/test/scala/server/SyncServer.scala @@ -1,16 +1,31 @@ package server +import datadog.trace.agent.test.base.HttpServerTest import datadog.trace.agent.test.base.HttpServerTest.ServerEndpoint._ import play.api.mvc.{Action, Handler, Results} import play.api.test.{FakeApplication, TestServer} object SyncServer { val routes: PartialFunction[(String, String), Handler] = { - case ("GET", "/success") => Action { request => Results.Status(SUCCESS.getStatus).apply(SUCCESS.getBody) } - case ("GET", "/redirect") => Action { request => Results.Redirect(REDIRECT.getBody, REDIRECT.getStatus) } - case ("GET", "/query") => Action { result => Results.Status(QUERY_PARAM.getStatus).apply(QUERY_PARAM.getBody) } - case ("GET", "/error-status") => Action { result => throw new RuntimeException(ERROR.getBody) } - case ("GET", "/exception") => Action { result => throw new RuntimeException(EXCEPTION.getBody) } + case ("GET", "/success") => Action { request => + HttpServerTest.controller(SUCCESS, new ControllerClosureAdapter(Results.Status(SUCCESS.getStatus).apply(SUCCESS.getBody))) + } + case ("GET", "/redirect") => Action { request => + HttpServerTest.controller(REDIRECT, new ControllerClosureAdapter(Results.Redirect(REDIRECT.getBody, REDIRECT.getStatus))) + } + case ("GET", "/query") => Action { request => + HttpServerTest.controller(QUERY_PARAM, new ControllerClosureAdapter(Results.Status(QUERY_PARAM.getStatus).apply(QUERY_PARAM.getBody))) + } + case ("GET", "/error-status") => Action { request => + HttpServerTest.controller(ERROR, new BlockClosureAdapter(() => { + throw new RuntimeException(ERROR.getBody) + })) + } + case ("GET", "/exception") => Action { request => + HttpServerTest.controller(EXCEPTION, new BlockClosureAdapter(() => { + throw new RuntimeException(EXCEPTION.getBody) + })) + } } def server(port: Int): TestServer = { diff --git a/dd-java-agent/testing/src/main/groovy/datadog/trace/agent/test/base/HttpServerTest.groovy b/dd-java-agent/testing/src/main/groovy/datadog/trace/agent/test/base/HttpServerTest.groovy index 16990954b1..3968ce78f6 100644 --- a/dd-java-agent/testing/src/main/groovy/datadog/trace/agent/test/base/HttpServerTest.groovy +++ b/dd-java-agent/testing/src/main/groovy/datadog/trace/agent/test/base/HttpServerTest.groovy @@ -24,12 +24,7 @@ import spock.lang.Unroll import java.util.concurrent.atomic.AtomicBoolean import static datadog.trace.agent.test.asserts.TraceAssert.assertTrace -import static datadog.trace.agent.test.base.HttpServerTest.ServerEndpoint.ERROR -import static datadog.trace.agent.test.base.HttpServerTest.ServerEndpoint.EXCEPTION -import static datadog.trace.agent.test.base.HttpServerTest.ServerEndpoint.NOT_FOUND -import static datadog.trace.agent.test.base.HttpServerTest.ServerEndpoint.QUERY_PARAM -import static datadog.trace.agent.test.base.HttpServerTest.ServerEndpoint.REDIRECT -import static datadog.trace.agent.test.base.HttpServerTest.ServerEndpoint.SUCCESS +import static datadog.trace.agent.test.base.HttpServerTest.ServerEndpoint.* import static datadog.trace.agent.test.utils.ConfigUtils.withConfigOverride import static datadog.trace.agent.test.utils.TraceUtils.basicSpan import static datadog.trace.agent.test.utils.TraceUtils.runUnderTrace @@ -533,7 +528,7 @@ abstract class HttpServerTest extends AgentTestRunner { tags { "$Tags.COMPONENT" component "$Tags.SPAN_KIND" Tags.SPAN_KIND_SERVER - "$Tags.PEER_PORT" Integer + "$Tags.PEER_PORT" { it == null || it instanceof Integer } // Optional "$Tags.PEER_HOST_IPV4" { it == null || it == "127.0.0.1" } // Optional "$Tags.HTTP_URL" "${endpoint.resolve(address)}" "$Tags.HTTP_METHOD" method diff --git a/dd-trace-ot/src/main/java/datadog/trace/common/writer/ListWriter.java b/dd-trace-ot/src/main/java/datadog/trace/common/writer/ListWriter.java index a92343350d..b1c8e06c68 100644 --- a/dd-trace-ot/src/main/java/datadog/trace/common/writer/ListWriter.java +++ b/dd-trace-ot/src/main/java/datadog/trace/common/writer/ListWriter.java @@ -39,6 +39,13 @@ public class ListWriter extends CopyOnWriteArrayList> implements Wr latches.add(latch); } if (!latch.await(20, TimeUnit.SECONDS)) { + System.out.println(this); + for (final List trace : this) { + System.out.println(trace.toString()); + for (final DDSpan span : trace) { + System.out.println(span.toString()); + } + } throw new TimeoutException( "Timeout waiting for " + number + " trace(s). ListWriter.size() == " + size()); } From 7f7b7d41cd5c4bdc1bce2f15cd2b63487518b300 Mon Sep 17 00:00:00 2001 From: Brian Devins-Suresh Date: Fri, 20 Mar 2020 06:51:22 -0400 Subject: [PATCH 18/43] Depend on netty 3.9 instrumentation --- .../instrumentation/play-2.3/play-2.3.gradle | 6 ++++-- .../test/groovy/server/PlayServerTest.groovy | 5 +++-- .../src/test/scala/server/AsyncServer.scala | 20 +++++++++---------- .../server/ControllerClosureAdapter.scala | 4 ++++ .../src/test/scala/server/SyncServer.scala | 6 ++---- .../agent/test/base/HttpServerTest.groovy | 9 +++++++-- .../trace/common/writer/ListWriter.java | 7 ------- 7 files changed, 29 insertions(+), 28 deletions(-) diff --git a/dd-java-agent/instrumentation/play-2.3/play-2.3.gradle b/dd-java-agent/instrumentation/play-2.3/play-2.3.gradle index 7614f1d01b..1bf70fd0c2 100644 --- a/dd-java-agent/instrumentation/play-2.3/play-2.3.gradle +++ b/dd-java-agent/instrumentation/play-2.3/play-2.3.gradle @@ -37,11 +37,13 @@ testSets { dependencies { main_java8Compile group: 'com.typesafe.play', name: 'play_2.11', version: '2.3.0' + testCompile project(':dd-java-agent:instrumentation:netty-3.9') + testCompile group: 'com.typesafe.play', name: 'play-java_2.11', version: '2.3.0' testCompile group: 'com.typesafe.play', name: 'play-java-ws_2.11', version: '2.3.0' - testCompile(group: 'com.typesafe.play', name: 'play-test_2.11', version: '2.3.0') + testCompile group: 'com.typesafe.play', name: 'play-test_2.11', version: '2.3.0' latestDepTestCompile group: 'com.typesafe.play', name: 'play-java_2.11', version: '2.3.+' latestDepTestCompile group: 'com.typesafe.play', name: 'play-java-ws_2.11', version: '2.3.+' - latestDepTestCompile(group: 'com.typesafe.play', name: 'play-test_2.11', version: '2.3.+') + latestDepTestCompile group: 'com.typesafe.play', name: 'play-test_2.11', version: '2.3.+' } diff --git a/dd-java-agent/instrumentation/play-2.3/src/test/groovy/server/PlayServerTest.groovy b/dd-java-agent/instrumentation/play-2.3/src/test/groovy/server/PlayServerTest.groovy index 32aa2e83c9..9267d0b9c7 100644 --- a/dd-java-agent/instrumentation/play-2.3/src/test/groovy/server/PlayServerTest.groovy +++ b/dd-java-agent/instrumentation/play-2.3/src/test/groovy/server/PlayServerTest.groovy @@ -6,6 +6,7 @@ import datadog.trace.agent.test.base.HttpServerTest import datadog.trace.api.DDSpanTypes import datadog.trace.api.DDTags import datadog.trace.bootstrap.instrumentation.api.Tags +import datadog.trace.instrumentation.netty39.server.NettyHttpServerDecorator import datadog.trace.instrumentation.play23.PlayHttpServerDecorator import play.api.test.TestServer @@ -26,12 +27,12 @@ class PlayServerTest extends HttpServerTest { @Override String component() { - return "netty" + return NettyHttpServerDecorator.DECORATE.component() } @Override String expectedOperationName() { - return "play.server" + return "netty.request" } // We don't have instrumentation for this version of netty yet diff --git a/dd-java-agent/instrumentation/play-2.3/src/test/scala/server/AsyncServer.scala b/dd-java-agent/instrumentation/play-2.3/src/test/scala/server/AsyncServer.scala index 087f520a85..a1789635a0 100644 --- a/dd-java-agent/instrumentation/play-2.3/src/test/scala/server/AsyncServer.scala +++ b/dd-java-agent/instrumentation/play-2.3/src/test/scala/server/AsyncServer.scala @@ -1,7 +1,7 @@ package server +import datadog.trace.agent.test.base.HttpServerTest import datadog.trace.agent.test.base.HttpServerTest.ServerEndpoint._ -import play.api.libs.concurrent.Execution.{defaultContext => ec} import play.api.mvc.{Action, Handler, Results} import play.api.test.{FakeApplication, TestServer} @@ -9,16 +9,14 @@ import scala.concurrent.Future object AsyncServer { val routes: PartialFunction[(String, String), Handler] = { - case ("GET", "/success") => Action.async { request => Future.successful(Results.Status(SUCCESS.getStatus).apply(SUCCESS.getBody)) } - case ("GET", "/redirect") => Action.async { request => Future.successful(Results.Redirect(REDIRECT.getBody, REDIRECT.getStatus)) } - case ("GET", "/query") => Action.async { result => Future.successful(Results.Status(QUERY_PARAM.getStatus).apply(QUERY_PARAM.getBody)) } - case ("GET", "/error-status") => Action.async { result => Future { - throw new RuntimeException(ERROR.getBody) - }(ec) - } - case ("GET", "/exception") => Action.async { result => Future { - throw new RuntimeException(ERROR.getBody) - }(ec) + case ("GET", "/success") => Action.async { request => HttpServerTest.controller(SUCCESS, new AsyncControllerClosureAdapter(Future.successful(Results.Status(SUCCESS.getStatus).apply(SUCCESS.getBody)))) } + case ("GET", "/redirect") => Action.async { request => HttpServerTest.controller(REDIRECT, new AsyncControllerClosureAdapter(Future.successful(Results.Redirect(REDIRECT.getBody, REDIRECT.getStatus)))) } + case ("GET", "/query") => Action.async { result => HttpServerTest.controller(QUERY_PARAM, new AsyncControllerClosureAdapter(Future.successful(Results.Status(QUERY_PARAM.getStatus).apply(QUERY_PARAM.getBody)))) } + case ("GET", "/error-status") => Action.async { result => HttpServerTest.controller(ERROR, new AsyncControllerClosureAdapter(Future.successful(Results.Status(ERROR.getStatus).apply(ERROR.getBody)))) } + case ("GET", "/exception") => Action.async { result => + HttpServerTest.controller(EXCEPTION, new AsyncBlockClosureAdapter(() => { + throw new Exception(EXCEPTION.getBody) + })) } } diff --git a/dd-java-agent/instrumentation/play-2.3/src/test/scala/server/ControllerClosureAdapter.scala b/dd-java-agent/instrumentation/play-2.3/src/test/scala/server/ControllerClosureAdapter.scala index bf800b1c87..8cf3766917 100644 --- a/dd-java-agent/instrumentation/play-2.3/src/test/scala/server/ControllerClosureAdapter.scala +++ b/dd-java-agent/instrumentation/play-2.3/src/test/scala/server/ControllerClosureAdapter.scala @@ -16,3 +16,7 @@ class BlockClosureAdapter(block: () => Result) extends Closure[Result] { class AsyncControllerClosureAdapter(response: Future[Result]) extends Closure[Future[Result]] { override def call(): Future[Result] = response } + +class AsyncBlockClosureAdapter(block: () => Future[Result]) extends Closure[Future[Result]] { + override def call(): Future[Result] = block() +} diff --git a/dd-java-agent/instrumentation/play-2.3/src/test/scala/server/SyncServer.scala b/dd-java-agent/instrumentation/play-2.3/src/test/scala/server/SyncServer.scala index fad6181a13..2565127ecb 100644 --- a/dd-java-agent/instrumentation/play-2.3/src/test/scala/server/SyncServer.scala +++ b/dd-java-agent/instrumentation/play-2.3/src/test/scala/server/SyncServer.scala @@ -17,13 +17,11 @@ object SyncServer { HttpServerTest.controller(QUERY_PARAM, new ControllerClosureAdapter(Results.Status(QUERY_PARAM.getStatus).apply(QUERY_PARAM.getBody))) } case ("GET", "/error-status") => Action { request => - HttpServerTest.controller(ERROR, new BlockClosureAdapter(() => { - throw new RuntimeException(ERROR.getBody) - })) + HttpServerTest.controller(QUERY_PARAM, new ControllerClosureAdapter(Results.Status(ERROR.getStatus).apply(ERROR.getBody))) } case ("GET", "/exception") => Action { request => HttpServerTest.controller(EXCEPTION, new BlockClosureAdapter(() => { - throw new RuntimeException(EXCEPTION.getBody) + throw new Exception(EXCEPTION.getBody) })) } } diff --git a/dd-java-agent/testing/src/main/groovy/datadog/trace/agent/test/base/HttpServerTest.groovy b/dd-java-agent/testing/src/main/groovy/datadog/trace/agent/test/base/HttpServerTest.groovy index 3968ce78f6..16990954b1 100644 --- a/dd-java-agent/testing/src/main/groovy/datadog/trace/agent/test/base/HttpServerTest.groovy +++ b/dd-java-agent/testing/src/main/groovy/datadog/trace/agent/test/base/HttpServerTest.groovy @@ -24,7 +24,12 @@ import spock.lang.Unroll import java.util.concurrent.atomic.AtomicBoolean import static datadog.trace.agent.test.asserts.TraceAssert.assertTrace -import static datadog.trace.agent.test.base.HttpServerTest.ServerEndpoint.* +import static datadog.trace.agent.test.base.HttpServerTest.ServerEndpoint.ERROR +import static datadog.trace.agent.test.base.HttpServerTest.ServerEndpoint.EXCEPTION +import static datadog.trace.agent.test.base.HttpServerTest.ServerEndpoint.NOT_FOUND +import static datadog.trace.agent.test.base.HttpServerTest.ServerEndpoint.QUERY_PARAM +import static datadog.trace.agent.test.base.HttpServerTest.ServerEndpoint.REDIRECT +import static datadog.trace.agent.test.base.HttpServerTest.ServerEndpoint.SUCCESS import static datadog.trace.agent.test.utils.ConfigUtils.withConfigOverride import static datadog.trace.agent.test.utils.TraceUtils.basicSpan import static datadog.trace.agent.test.utils.TraceUtils.runUnderTrace @@ -528,7 +533,7 @@ abstract class HttpServerTest extends AgentTestRunner { tags { "$Tags.COMPONENT" component "$Tags.SPAN_KIND" Tags.SPAN_KIND_SERVER - "$Tags.PEER_PORT" { it == null || it instanceof Integer } // Optional + "$Tags.PEER_PORT" Integer "$Tags.PEER_HOST_IPV4" { it == null || it == "127.0.0.1" } // Optional "$Tags.HTTP_URL" "${endpoint.resolve(address)}" "$Tags.HTTP_METHOD" method diff --git a/dd-trace-ot/src/main/java/datadog/trace/common/writer/ListWriter.java b/dd-trace-ot/src/main/java/datadog/trace/common/writer/ListWriter.java index b1c8e06c68..a92343350d 100644 --- a/dd-trace-ot/src/main/java/datadog/trace/common/writer/ListWriter.java +++ b/dd-trace-ot/src/main/java/datadog/trace/common/writer/ListWriter.java @@ -39,13 +39,6 @@ public class ListWriter extends CopyOnWriteArrayList> implements Wr latches.add(latch); } if (!latch.await(20, TimeUnit.SECONDS)) { - System.out.println(this); - for (final List trace : this) { - System.out.println(trace.toString()); - for (final DDSpan span : trace) { - System.out.println(span.toString()); - } - } throw new TimeoutException( "Timeout waiting for " + number + " trace(s). ListWriter.size() == " + size()); } From 879d84a162d71b051aed4e64eda5ac3b39850d96 Mon Sep 17 00:00:00 2001 From: Brian Devins-Suresh Date: Fri, 20 Mar 2020 07:58:55 -0400 Subject: [PATCH 19/43] Try to fix client test, not quite --- .../instrumentation/play-2.3/play-2.3.gradle | 8 ++++-- .../groovy/client/PlayWSClientTest.groovy | 26 ++++++++++++++----- 2 files changed, 26 insertions(+), 8 deletions(-) diff --git a/dd-java-agent/instrumentation/play-2.3/play-2.3.gradle b/dd-java-agent/instrumentation/play-2.3/play-2.3.gradle index 1bf70fd0c2..1e5c01151b 100644 --- a/dd-java-agent/instrumentation/play-2.3/play-2.3.gradle +++ b/dd-java-agent/instrumentation/play-2.3/play-2.3.gradle @@ -41,9 +41,13 @@ dependencies { testCompile group: 'com.typesafe.play', name: 'play-java_2.11', version: '2.3.0' testCompile group: 'com.typesafe.play', name: 'play-java-ws_2.11', version: '2.3.0' - testCompile group: 'com.typesafe.play', name: 'play-test_2.11', version: '2.3.0' + testCompile(group: 'com.typesafe.play', name: 'play-test_2.11', version: '2.3.0') { + exclude group: 'org.eclipse.jetty', module: 'jetty-websocket' + } latestDepTestCompile group: 'com.typesafe.play', name: 'play-java_2.11', version: '2.3.+' latestDepTestCompile group: 'com.typesafe.play', name: 'play-java-ws_2.11', version: '2.3.+' - latestDepTestCompile group: 'com.typesafe.play', name: 'play-test_2.11', version: '2.3.+' + latestDepTestCompile(group: 'com.typesafe.play', name: 'play-test_2.11', version: '2.3.+') { + exclude group: 'org.eclipse.jetty', module: 'jetty-websocket' + } } diff --git a/dd-java-agent/instrumentation/play-2.3/src/test/groovy/client/PlayWSClientTest.groovy b/dd-java-agent/instrumentation/play-2.3/src/test/groovy/client/PlayWSClientTest.groovy index 2e0b8bfce9..e9165f086b 100644 --- a/dd-java-agent/instrumentation/play-2.3/src/test/groovy/client/PlayWSClientTest.groovy +++ b/dd-java-agent/instrumentation/play-2.3/src/test/groovy/client/PlayWSClientTest.groovy @@ -1,21 +1,35 @@ package client import datadog.trace.agent.test.base.HttpClientTest +import datadog.trace.instrumentation.netty39.client.NettyHttpClientDecorator +import play.GlobalSettings import play.libs.ws.WS -import spock.lang.AutoCleanup +import play.test.FakeApplication +import play.test.Helpers import spock.lang.Shared -import spock.lang.Subject import java.util.concurrent.TimeUnit class PlayWSClientTest extends HttpClientTest { - @Subject @Shared - @AutoCleanup - def client = WS.client() + def application = new FakeApplication(new File("."), FakeApplication.class.getClassLoader(), Collections.emptyMap(), Collections.emptyList(), new GlobalSettings()) + +// @Subject +// @Shared +// @AutoCleanup +// def client = WS.client() + + def setup() { + Helpers.start(application) + } + + def cleanup() { + Helpers.stop(application) + } @Override int doRequest(String method, URI uri, Map headers, Closure callback) { + def client = WS.client() def request = client.url(uri.toString()) headers.entrySet().each { request.setHeader(it.key, it.value) @@ -32,7 +46,7 @@ class PlayWSClientTest extends HttpClientTest { @Override String component() { - return "" // NettyHttpClientDecorator.DECORATE.component() + return NettyHttpClientDecorator.DECORATE.component() } @Override From 6fa94ed9ac32ef275a89688a22d0ba33366f0342 Mon Sep 17 00:00:00 2001 From: Brian Devins-Suresh Date: Fri, 20 Mar 2020 15:14:34 -0400 Subject: [PATCH 20/43] Update to new module name --- dd-java-agent/instrumentation/play-2.3/play-2.3.gradle | 2 +- .../src/test/groovy/client/PlayWSClientTest.groovy | 7 +------ .../play-2.3/src/test/groovy/server/PlayServerTest.groovy | 2 +- 3 files changed, 3 insertions(+), 8 deletions(-) diff --git a/dd-java-agent/instrumentation/play-2.3/play-2.3.gradle b/dd-java-agent/instrumentation/play-2.3/play-2.3.gradle index 1e5c01151b..fc88656192 100644 --- a/dd-java-agent/instrumentation/play-2.3/play-2.3.gradle +++ b/dd-java-agent/instrumentation/play-2.3/play-2.3.gradle @@ -37,7 +37,7 @@ testSets { dependencies { main_java8Compile group: 'com.typesafe.play', name: 'play_2.11', version: '2.3.0' - testCompile project(':dd-java-agent:instrumentation:netty-3.9') + testCompile project(':dd-java-agent:instrumentation:netty-3.8') testCompile group: 'com.typesafe.play', name: 'play-java_2.11', version: '2.3.0' testCompile group: 'com.typesafe.play', name: 'play-java-ws_2.11', version: '2.3.0' diff --git a/dd-java-agent/instrumentation/play-2.3/src/test/groovy/client/PlayWSClientTest.groovy b/dd-java-agent/instrumentation/play-2.3/src/test/groovy/client/PlayWSClientTest.groovy index e9165f086b..e254806bb4 100644 --- a/dd-java-agent/instrumentation/play-2.3/src/test/groovy/client/PlayWSClientTest.groovy +++ b/dd-java-agent/instrumentation/play-2.3/src/test/groovy/client/PlayWSClientTest.groovy @@ -1,7 +1,7 @@ package client import datadog.trace.agent.test.base.HttpClientTest -import datadog.trace.instrumentation.netty39.client.NettyHttpClientDecorator +import datadog.trace.instrumentation.netty38.client.NettyHttpClientDecorator import play.GlobalSettings import play.libs.ws.WS import play.test.FakeApplication @@ -14,11 +14,6 @@ class PlayWSClientTest extends HttpClientTest { @Shared def application = new FakeApplication(new File("."), FakeApplication.class.getClassLoader(), Collections.emptyMap(), Collections.emptyList(), new GlobalSettings()) -// @Subject -// @Shared -// @AutoCleanup -// def client = WS.client() - def setup() { Helpers.start(application) } diff --git a/dd-java-agent/instrumentation/play-2.3/src/test/groovy/server/PlayServerTest.groovy b/dd-java-agent/instrumentation/play-2.3/src/test/groovy/server/PlayServerTest.groovy index 9267d0b9c7..c8ce2c32c5 100644 --- a/dd-java-agent/instrumentation/play-2.3/src/test/groovy/server/PlayServerTest.groovy +++ b/dd-java-agent/instrumentation/play-2.3/src/test/groovy/server/PlayServerTest.groovy @@ -6,7 +6,7 @@ import datadog.trace.agent.test.base.HttpServerTest import datadog.trace.api.DDSpanTypes import datadog.trace.api.DDTags import datadog.trace.bootstrap.instrumentation.api.Tags -import datadog.trace.instrumentation.netty39.server.NettyHttpServerDecorator +import datadog.trace.instrumentation.netty38.server.NettyHttpServerDecorator import datadog.trace.instrumentation.play23.PlayHttpServerDecorator import play.api.test.TestServer From 661a8f5a3ecae74f56ae768750f7af7fa31b2e1a Mon Sep 17 00:00:00 2001 From: Brian Devins-Suresh Date: Mon, 23 Mar 2020 14:01:57 -0400 Subject: [PATCH 21/43] Fix the latestdeptests --- dd-java-agent/instrumentation/play-2.3/play-2.3.gradle | 5 +++++ .../src/test/groovy/client/PlayWSClientTest.groovy | 8 +++++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/dd-java-agent/instrumentation/play-2.3/play-2.3.gradle b/dd-java-agent/instrumentation/play-2.3/play-2.3.gradle index fc88656192..4e74546bc0 100644 --- a/dd-java-agent/instrumentation/play-2.3/play-2.3.gradle +++ b/dd-java-agent/instrumentation/play-2.3/play-2.3.gradle @@ -51,3 +51,8 @@ dependencies { exclude group: 'org.eclipse.jetty', module: 'jetty-websocket' } } + +compileLatestDepTestGroovy { + classpath = classpath.plus(files(compileLatestDepTestScala.destinationDir)) + dependsOn compileLatestDepTestScala +} diff --git a/dd-java-agent/instrumentation/play-2.3/src/test/groovy/client/PlayWSClientTest.groovy b/dd-java-agent/instrumentation/play-2.3/src/test/groovy/client/PlayWSClientTest.groovy index e254806bb4..9d8c119f2c 100644 --- a/dd-java-agent/instrumentation/play-2.3/src/test/groovy/client/PlayWSClientTest.groovy +++ b/dd-java-agent/instrumentation/play-2.3/src/test/groovy/client/PlayWSClientTest.groovy @@ -12,7 +12,13 @@ import java.util.concurrent.TimeUnit class PlayWSClientTest extends HttpClientTest { @Shared - def application = new FakeApplication(new File("."), FakeApplication.class.getClassLoader(), Collections.emptyMap(), Collections.emptyList(), new GlobalSettings()) + def application = new FakeApplication( + new File("."), + FakeApplication.class.getClassLoader(), + Collections.emptyMap(), + Collections.emptyList(), + new GlobalSettings() + ) def setup() { Helpers.start(application) From 8dd5993a8fb74c890cc1442f74edd3feed0ff1a3 Mon Sep 17 00:00:00 2001 From: Brian Devins-Suresh Date: Mon, 23 Mar 2020 15:08:04 -0400 Subject: [PATCH 22/43] Fix codenarc --- .../play-2.3/src/test/groovy/client/PlayWSClientTest.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dd-java-agent/instrumentation/play-2.3/src/test/groovy/client/PlayWSClientTest.groovy b/dd-java-agent/instrumentation/play-2.3/src/test/groovy/client/PlayWSClientTest.groovy index 9d8c119f2c..12ddf851b3 100644 --- a/dd-java-agent/instrumentation/play-2.3/src/test/groovy/client/PlayWSClientTest.groovy +++ b/dd-java-agent/instrumentation/play-2.3/src/test/groovy/client/PlayWSClientTest.groovy @@ -14,7 +14,7 @@ class PlayWSClientTest extends HttpClientTest { @Shared def application = new FakeApplication( new File("."), - FakeApplication.class.getClassLoader(), + FakeApplication.getClassLoader(), Collections.emptyMap(), Collections.emptyList(), new GlobalSettings() From d3def8ce2d892ff2a6a49a6e4767c8acfc619e94 Mon Sep 17 00:00:00 2001 From: Brian Devins-Suresh Date: Mon, 23 Mar 2020 15:11:10 -0400 Subject: [PATCH 23/43] :( --- .../play-2.3/src/test/groovy/client/PlayWSClientTest.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dd-java-agent/instrumentation/play-2.3/src/test/groovy/client/PlayWSClientTest.groovy b/dd-java-agent/instrumentation/play-2.3/src/test/groovy/client/PlayWSClientTest.groovy index 12ddf851b3..fd7085437d 100644 --- a/dd-java-agent/instrumentation/play-2.3/src/test/groovy/client/PlayWSClientTest.groovy +++ b/dd-java-agent/instrumentation/play-2.3/src/test/groovy/client/PlayWSClientTest.groovy @@ -20,11 +20,11 @@ class PlayWSClientTest extends HttpClientTest { new GlobalSettings() ) - def setup() { + def setupSpec() { Helpers.start(application) } - def cleanup() { + def cleanupSpec() { Helpers.stop(application) } From acba3626dce035df543098252feabca06d8dce14 Mon Sep 17 00:00:00 2001 From: Brian Devins-Suresh Date: Mon, 23 Mar 2020 15:59:06 -0400 Subject: [PATCH 24/43] Missed a static import --- .../java8/datadog/trace/instrumentation/play23/PlayAdvice.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dd-java-agent/instrumentation/play-2.3/src/main/java8/datadog/trace/instrumentation/play23/PlayAdvice.java b/dd-java-agent/instrumentation/play-2.3/src/main/java8/datadog/trace/instrumentation/play23/PlayAdvice.java index 5c1de38c6b..7108037f96 100644 --- a/dd-java-agent/instrumentation/play-2.3/src/main/java8/datadog/trace/instrumentation/play23/PlayAdvice.java +++ b/dd-java-agent/instrumentation/play-2.3/src/main/java8/datadog/trace/instrumentation/play23/PlayAdvice.java @@ -4,6 +4,7 @@ import static datadog.trace.bootstrap.instrumentation.api.AgentTracer.activateSp import static datadog.trace.bootstrap.instrumentation.api.AgentTracer.activeSpan; import static datadog.trace.bootstrap.instrumentation.api.AgentTracer.propagate; import static datadog.trace.bootstrap.instrumentation.api.AgentTracer.startSpan; +import static datadog.trace.instrumentation.play23.PlayHeaders.GETTER; import static datadog.trace.instrumentation.play23.PlayHttpServerDecorator.DECORATE; import datadog.trace.bootstrap.instrumentation.api.AgentScope; @@ -21,7 +22,7 @@ public class PlayAdvice { public static AgentScope onEnter(@Advice.Argument(0) final Request req) { final AgentSpan span; if (activeSpan() == null) { - final Context extractedContext = propagate().extract(req.headers(), PlayHeaders.GETTER); + final Context extractedContext = propagate().extract(req.headers(), GETTER); span = startSpan("play.request", extractedContext); } else { // An upstream framework (e.g. akka-http, netty) has already started the span. From 08e9f8c6e2a1b5d379ca07160b67b6960a783aae Mon Sep 17 00:00:00 2001 From: Brian Devins-Suresh Date: Mon, 23 Mar 2020 16:02:02 -0400 Subject: [PATCH 25/43] Undo star import --- buildSrc/src/main/groovy/MuzzlePlugin.groovy | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/buildSrc/src/main/groovy/MuzzlePlugin.groovy b/buildSrc/src/main/groovy/MuzzlePlugin.groovy index d8180344d8..71420ccb22 100644 --- a/buildSrc/src/main/groovy/MuzzlePlugin.groovy +++ b/buildSrc/src/main/groovy/MuzzlePlugin.groovy @@ -14,7 +14,11 @@ import org.eclipse.aether.spi.connector.RepositoryConnectorFactory import org.eclipse.aether.spi.connector.transport.TransporterFactory import org.eclipse.aether.transport.http.HttpTransporterFactory import org.eclipse.aether.version.Version -import org.gradle.api.* +import org.gradle.api.Action +import org.gradle.api.GradleException +import org.gradle.api.Plugin +import org.gradle.api.Project +import org.gradle.api.Task import org.gradle.api.model.ObjectFactory import java.lang.reflect.Method From 026072b866806b2acd207bda83f147e2f785b138 Mon Sep 17 00:00:00 2001 From: Brian Devins-Suresh Date: Wed, 25 Mar 2020 12:57:00 -0400 Subject: [PATCH 26/43] Wrong endpoint --- .../play-2.3/src/test/scala/server/SyncServer.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dd-java-agent/instrumentation/play-2.3/src/test/scala/server/SyncServer.scala b/dd-java-agent/instrumentation/play-2.3/src/test/scala/server/SyncServer.scala index 2565127ecb..602a14c70e 100644 --- a/dd-java-agent/instrumentation/play-2.3/src/test/scala/server/SyncServer.scala +++ b/dd-java-agent/instrumentation/play-2.3/src/test/scala/server/SyncServer.scala @@ -17,7 +17,7 @@ object SyncServer { HttpServerTest.controller(QUERY_PARAM, new ControllerClosureAdapter(Results.Status(QUERY_PARAM.getStatus).apply(QUERY_PARAM.getBody))) } case ("GET", "/error-status") => Action { request => - HttpServerTest.controller(QUERY_PARAM, new ControllerClosureAdapter(Results.Status(ERROR.getStatus).apply(ERROR.getBody))) + HttpServerTest.controller(ERROR, new ControllerClosureAdapter(Results.Status(ERROR.getStatus).apply(ERROR.getBody))) } case ("GET", "/exception") => Action { request => HttpServerTest.controller(EXCEPTION, new BlockClosureAdapter(() => { From fbb43840d39ee5385413505e623caf175656e684 Mon Sep 17 00:00:00 2001 From: Brian Devins-Suresh Date: Thu, 26 Mar 2020 14:29:35 -0400 Subject: [PATCH 27/43] Fix new client tests --- .../groovy/client/PlayWSClientTest.groovy | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/dd-java-agent/instrumentation/play-2.3/src/test/groovy/client/PlayWSClientTest.groovy b/dd-java-agent/instrumentation/play-2.3/src/test/groovy/client/PlayWSClientTest.groovy index fd7085437d..a72b00c7b3 100644 --- a/dd-java-agent/instrumentation/play-2.3/src/test/groovy/client/PlayWSClientTest.groovy +++ b/dd-java-agent/instrumentation/play-2.3/src/test/groovy/client/PlayWSClientTest.groovy @@ -15,13 +15,20 @@ class PlayWSClientTest extends HttpClientTest { def application = new FakeApplication( new File("."), FakeApplication.getClassLoader(), - Collections.emptyMap(), + [ + "ws.timeout.connection": CONNECT_TIMEOUT_MS, + "ws.timeout.request" : READ_TIMEOUT_MS + ], Collections.emptyList(), new GlobalSettings() ) + @Shared + def client + def setupSpec() { Helpers.start(application) + client = WS.client() } def cleanupSpec() { @@ -30,7 +37,6 @@ class PlayWSClientTest extends HttpClientTest { @Override int doRequest(String method, URI uri, Map headers, Closure callback) { - def client = WS.client() def request = client.url(uri.toString()) headers.entrySet().each { request.setHeader(it.key, it.value) @@ -64,4 +70,13 @@ class PlayWSClientTest extends HttpClientTest { boolean testConnectionFailure() { false } + + @Override + boolean testRemoteConnection() { + // On connection failures the operation and resource names end up different from expected. + // This would require a lot of changes to the base client test class to support + // span.operationName = "netty.connect" + // span.resourceName = "netty.connect" + false + } } From 96757f0c58432f982976340c0622b17c3e10d124 Mon Sep 17 00:00:00 2001 From: jean-philippe bempel Date: Thu, 2 Apr 2020 17:54:02 +0200 Subject: [PATCH 28/43] Remove sensitive information from debug log Config.toString() method is dumped when logging in debug the conf. It includes in some case the profile api key when used with env vars. Also proxy password is also dumped. toString method generated by Lombok now excludes both fields --- .../src/main/java/datadog/trace/api/Config.java | 2 +- .../test/groovy/datadog/trace/api/ConfigTest.groovy | 13 +++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/dd-trace-api/src/main/java/datadog/trace/api/Config.java b/dd-trace-api/src/main/java/datadog/trace/api/Config.java index e9b6e9c816..72ed152750 100644 --- a/dd-trace-api/src/main/java/datadog/trace/api/Config.java +++ b/dd-trace-api/src/main/java/datadog/trace/api/Config.java @@ -38,7 +38,7 @@ import lombok.extern.slf4j.Slf4j; * system property, but uppercased with '.' -> '_'. */ @Slf4j -@ToString(includeFieldNames = true) +@ToString(includeFieldNames = true, exclude = {"profilingApiKey", "profilingProxyPassword"}) public class Config { /** Config keys below */ private static final String PREFIX = "dd."; diff --git a/dd-trace-api/src/test/groovy/datadog/trace/api/ConfigTest.groovy b/dd-trace-api/src/test/groovy/datadog/trace/api/ConfigTest.groovy index db37f07a6a..82e29caab6 100644 --- a/dd-trace-api/src/test/groovy/datadog/trace/api/ConfigTest.groovy +++ b/dd-trace-api/src/test/groovy/datadog/trace/api/ConfigTest.groovy @@ -406,6 +406,19 @@ class ConfigTest extends DDSpecification { config.profilingApiKey == "test-api-key" } + def "sensitive information removed for toString/debug log"() { + setup: + environmentVariables.set(DD_PROFILING_API_KEY_ENV, "test-secret-api-key") + environmentVariables.set(PROFILING_PROXY_PASSWORD, "test-secret-proxy-password") + + when: + def config = new Config() + + then: + !config.toString().contains("test-secret-api-key") + !config.toString().contains("test-secret-proxy-password") + } + def "sys props override env vars"() { setup: environmentVariables.set(DD_SERVICE_NAME_ENV, "still something else") From 4d15d5ae968f0bf00df95acd7dc7b1f69bd97cf9 Mon Sep 17 00:00:00 2001 From: jean-philippe bempel Date: Thu, 2 Apr 2020 18:26:14 +0200 Subject: [PATCH 29/43] Sensor information instead of excluding the fields --- .../src/main/java/datadog/trace/api/Config.java | 10 +++++++++- .../test/groovy/datadog/trace/api/ConfigTest.groovy | 2 ++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/dd-trace-api/src/main/java/datadog/trace/api/Config.java b/dd-trace-api/src/main/java/datadog/trace/api/Config.java index 72ed152750..38fdc20eff 100644 --- a/dd-trace-api/src/main/java/datadog/trace/api/Config.java +++ b/dd-trace-api/src/main/java/datadog/trace/api/Config.java @@ -38,7 +38,7 @@ import lombok.extern.slf4j.Slf4j; * system property, but uppercased with '.' -> '_'. */ @Slf4j -@ToString(includeFieldNames = true, exclude = {"profilingApiKey", "profilingProxyPassword"}) +@ToString(includeFieldNames = true) public class Config { /** Config keys below */ private static final String PREFIX = "dd."; @@ -202,6 +202,14 @@ public class Config { /** A tag intended for internal use only, hence not added to the public api DDTags class. */ private static final String INTERNAL_HOST_NAME = "_dd.hostname"; + /** Used for masking sensitive information when doing toString */ + @ToString.Include(name = "profilingApiKey") + private String profilingApiKeyMasker() { return "****"; } + + /** Used for masking sensitive information when doing toString */ + @ToString.Include(name = "profilingProxyPassword") + private String profilingProxyPasswordMasker() { return "****"; } + /** * this is a random UUID that gets generated on JVM start up and is attached to every root span * and every JMX metric that is sent out. diff --git a/dd-trace-api/src/test/groovy/datadog/trace/api/ConfigTest.groovy b/dd-trace-api/src/test/groovy/datadog/trace/api/ConfigTest.groovy index 82e29caab6..5959849288 100644 --- a/dd-trace-api/src/test/groovy/datadog/trace/api/ConfigTest.groovy +++ b/dd-trace-api/src/test/groovy/datadog/trace/api/ConfigTest.groovy @@ -415,7 +415,9 @@ class ConfigTest extends DDSpecification { def config = new Config() then: + config.toString().contains("profilingApiKey=****"); !config.toString().contains("test-secret-api-key") + config.toString().contains("profilingProxyPassword=****"); !config.toString().contains("test-secret-proxy-password") } From 58ea15e590baca4aa5894cd3033b0c4894dcf759 Mon Sep 17 00:00:00 2001 From: jean-philippe bempel Date: Thu, 2 Apr 2020 19:17:26 +0200 Subject: [PATCH 30/43] if not populated returns null for toString --- dd-trace-api/src/main/java/datadog/trace/api/Config.java | 8 ++++++-- .../src/test/groovy/datadog/trace/api/ConfigTest.groovy | 5 ++++- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/dd-trace-api/src/main/java/datadog/trace/api/Config.java b/dd-trace-api/src/main/java/datadog/trace/api/Config.java index 38fdc20eff..430b01adf0 100644 --- a/dd-trace-api/src/main/java/datadog/trace/api/Config.java +++ b/dd-trace-api/src/main/java/datadog/trace/api/Config.java @@ -204,11 +204,15 @@ public class Config { /** Used for masking sensitive information when doing toString */ @ToString.Include(name = "profilingApiKey") - private String profilingApiKeyMasker() { return "****"; } + private String profilingApiKeyMasker() { + return profilingApiKey != null ? "****" : null; + } /** Used for masking sensitive information when doing toString */ @ToString.Include(name = "profilingProxyPassword") - private String profilingProxyPasswordMasker() { return "****"; } + private String profilingProxyPasswordMasker() { + return profilingProxyPassword != null ? "****" : null; + } /** * this is a random UUID that gets generated on JVM start up and is attached to every root span diff --git a/dd-trace-api/src/test/groovy/datadog/trace/api/ConfigTest.groovy b/dd-trace-api/src/test/groovy/datadog/trace/api/ConfigTest.groovy index 5959849288..4eb3ef7ad3 100644 --- a/dd-trace-api/src/test/groovy/datadog/trace/api/ConfigTest.groovy +++ b/dd-trace-api/src/test/groovy/datadog/trace/api/ConfigTest.groovy @@ -93,6 +93,7 @@ class ConfigTest extends DDSpecification { private static final DD_PROFILING_API_KEY_ENV = "DD_PROFILING_API_KEY" private static final DD_PROFILING_API_KEY_OLD_ENV = "DD_PROFILING_APIKEY" private static final DD_PROFILING_TAGS_ENV = "DD_PROFILING_TAGS" + private static final DD_PROFILING_PROXY_PASSWORD_ENV = "DD_PROFILING_PROXY_PASSWORD" def "verify defaults"() { when: @@ -409,7 +410,7 @@ class ConfigTest extends DDSpecification { def "sensitive information removed for toString/debug log"() { setup: environmentVariables.set(DD_PROFILING_API_KEY_ENV, "test-secret-api-key") - environmentVariables.set(PROFILING_PROXY_PASSWORD, "test-secret-proxy-password") + environmentVariables.set(DD_PROFILING_PROXY_PASSWORD_ENV, "test-secret-proxy-password") when: def config = new Config() @@ -419,6 +420,8 @@ class ConfigTest extends DDSpecification { !config.toString().contains("test-secret-api-key") config.toString().contains("profilingProxyPassword=****"); !config.toString().contains("test-secret-proxy-password") + config.getProfilingApiKey() == "test-secret-api-key" + config.getProfilingProxyPassword() == "test-secret-proxy-password" } def "sys props override env vars"() { From d1f1f8d6e9a1bc2abd04af9a13a54465b7118ff8 Mon Sep 17 00:00:00 2001 From: Nikolay Martynov Date: Thu, 2 Apr 2020 15:55:24 -0400 Subject: [PATCH 31/43] Add option to configure Datadog site used to send profiling information --- .../profiling/uploader/RecordingUploader.java | 4 ++- .../uploader/RecordingUploaderTest.java | 6 ++-- .../main/java/datadog/trace/api/Config.java | 34 +++++++++++++------ .../datadog/trace/api/ConfigTest.groovy | 33 +++++++++++++++++- 4 files changed, 62 insertions(+), 15 deletions(-) diff --git a/dd-java-agent/agent-profiling/profiling-uploader/src/main/java/com/datadog/profiling/uploader/RecordingUploader.java b/dd-java-agent/agent-profiling/profiling-uploader/src/main/java/com/datadog/profiling/uploader/RecordingUploader.java index e92afe9eda..5be5bf3974 100644 --- a/dd-java-agent/agent-profiling/profiling-uploader/src/main/java/com/datadog/profiling/uploader/RecordingUploader.java +++ b/dd-java-agent/agent-profiling/profiling-uploader/src/main/java/com/datadog/profiling/uploader/RecordingUploader.java @@ -53,6 +53,7 @@ import okhttp3.Response; /** The class for uploading recordings to the backend. */ @Slf4j public final class RecordingUploader { + private static final MediaType OCTET_STREAM = MediaType.parse("application/octet-stream"); static final String RECORDING_NAME_PARAM = "recording-name"; @@ -120,7 +121,7 @@ public final class RecordingUploader { private final Deque requestSizeHistory; public RecordingUploader(final Config config) { - url = config.getProfilingUrl(); + url = config.getFinalProfilingUrl(); apiKey = config.getProfilingApiKey(); /* @@ -233,6 +234,7 @@ public final class RecordingUploader { @FunctionalInterface private interface Compression { + RequestBody compress(InputStream is, int expectedSize) throws IOException; } diff --git a/dd-java-agent/agent-profiling/profiling-uploader/src/test/java/com/datadog/profiling/uploader/RecordingUploaderTest.java b/dd-java-agent/agent-profiling/profiling-uploader/src/test/java/com/datadog/profiling/uploader/RecordingUploaderTest.java index f550e267aa..77b492e395 100644 --- a/dd-java-agent/agent-profiling/profiling-uploader/src/test/java/com/datadog/profiling/uploader/RecordingUploaderTest.java +++ b/dd-java-agent/agent-profiling/profiling-uploader/src/test/java/com/datadog/profiling/uploader/RecordingUploaderTest.java @@ -120,7 +120,7 @@ public class RecordingUploaderTest { server.start(); url = server.url(URL_PATH); - when(config.getProfilingUrl()).thenReturn(server.url(URL_PATH).toString()); + when(config.getFinalProfilingUrl()).thenReturn(server.url(URL_PATH).toString()); when(config.getProfilingApiKey()).thenReturn(APIKEY_VALUE); when(config.getMergedProfilingTags()).thenReturn(TAGS); when(config.getProfilingUploadTimeout()).thenReturn((int) REQUEST_TIMEOUT.getSeconds()); @@ -199,7 +199,7 @@ public class RecordingUploaderTest { public void testRequestWithProxy() throws IOException, InterruptedException { final String backendHost = "intake.profiling.datadoghq.com:1234"; final String backendUrl = "http://intake.profiling.datadoghq.com:1234" + URL_PATH; - when(config.getProfilingUrl()) + when(config.getFinalProfilingUrl()) .thenReturn("http://intake.profiling.datadoghq.com:1234" + URL_PATH); when(config.getProfilingProxyHost()).thenReturn(server.url("").host()); when(config.getProfilingProxyPort()).thenReturn(server.url("").port()); @@ -235,7 +235,7 @@ public class RecordingUploaderTest { @Test public void testRequestWithProxyDefaultPassword() throws IOException, InterruptedException { final String backendUrl = "http://intake.profiling.datadoghq.com:1234" + URL_PATH; - when(config.getProfilingUrl()) + when(config.getFinalProfilingUrl()) .thenReturn("http://intake.profiling.datadoghq.com:1234" + URL_PATH); when(config.getProfilingProxyHost()).thenReturn(server.url("").host()); when(config.getProfilingProxyPort()).thenReturn(server.url("").port()); diff --git a/dd-trace-api/src/main/java/datadog/trace/api/Config.java b/dd-trace-api/src/main/java/datadog/trace/api/Config.java index e9b6e9c816..b645cb5fa8 100644 --- a/dd-trace-api/src/main/java/datadog/trace/api/Config.java +++ b/dd-trace-api/src/main/java/datadog/trace/api/Config.java @@ -40,9 +40,12 @@ import lombok.extern.slf4j.Slf4j; @Slf4j @ToString(includeFieldNames = true) public class Config { + /** Config keys below */ private static final String PREFIX = "dd."; + public static final String PROFILING_URL_TEMPLATE = "https://intake.profile.%s/v1/input"; + private static final Pattern ENV_REPLACEMENT = Pattern.compile("[^a-zA-Z0-9_]"); public static final String CONFIGURATION_FILE = "trace.config"; @@ -106,6 +109,7 @@ public class Config { public static final String LOGS_INJECTION_ENABLED = "logs.injection"; public static final String PROFILING_ENABLED = "profiling.enabled"; + public static final String PROFILING_SITE = "profiling.site"; public static final String PROFILING_URL = "profiling.url"; public static final String PROFILING_API_KEY = "profiling.api-key"; @@ -173,8 +177,7 @@ public class Config { public static final boolean DEFAULT_LOGS_INJECTION_ENABLED = false; public static final boolean DEFAULT_PROFILING_ENABLED = false; - public static final String DEFAULT_PROFILING_URL = - "https://intake.profile.datadoghq.com/v1/input"; + public static final String DEFAULT_PROFILING_SITE = "datadoghq.com"; public static final int DEFAULT_PROFILING_START_DELAY = 10; public static final boolean DEFAULT_PROFILING_START_FORCE_FIRST = false; public static final int DEFAULT_PROFILING_UPLOAD_PERIOD = 60; // 1 min @@ -269,7 +272,8 @@ public class Config { @Getter private final Double traceRateLimit; @Getter private final boolean profilingEnabled; - @Getter private final String profilingUrl; + @Getter private final String profilingSite; + private final String profilingUrl; @Getter private final String profilingApiKey; private final Map profilingTags; @Getter private final int profilingStartDelay; @@ -415,7 +419,8 @@ public class Config { profilingEnabled = getBooleanSettingFromEnvironment(PROFILING_ENABLED, DEFAULT_PROFILING_ENABLED); - profilingUrl = getSettingFromEnvironment(PROFILING_URL, DEFAULT_PROFILING_URL); + profilingSite = getSettingFromEnvironment(PROFILING_SITE, DEFAULT_PROFILING_SITE); + profilingUrl = getSettingFromEnvironment(PROFILING_URL, null); // Note: We do not want APiKey to be loaded from property for security reasons // Note: we do not use defined default here // FIXME: We should use better authentication mechanism @@ -609,6 +614,7 @@ public class Config { profilingEnabled = getPropertyBooleanValue(properties, PROFILING_ENABLED, parent.profilingEnabled); + profilingSite = properties.getProperty(PROFILING_SITE, parent.profilingSite); profilingUrl = properties.getProperty(PROFILING_URL, parent.profilingUrl); profilingApiKey = properties.getProperty(PROFILING_API_KEY, parent.profilingApiKey); profilingTags = getPropertyMapValue(properties, PROFILING_TAGS, parent.profilingTags); @@ -734,17 +740,25 @@ public class Config { return Collections.unmodifiableMap(result); } + public String getFinalProfilingUrl() { + if (profilingUrl == null) { + return String.format(PROFILING_URL_TEMPLATE, profilingSite); + } else { + return profilingUrl; + } + } + public boolean isIntegrationEnabled( final SortedSet integrationNames, final boolean defaultEnabled) { return integrationEnabled(integrationNames, defaultEnabled); } /** - * @deprecated This method should only be used internally. Use the instance getter instead {@link - * #isIntegrationEnabled(SortedSet, boolean)}. * @param integrationNames * @param defaultEnabled * @return + * @deprecated This method should only be used internally. Use the instance getter instead {@link + * #isIntegrationEnabled(SortedSet, boolean)}. */ public static boolean integrationEnabled( final SortedSet integrationNames, final boolean defaultEnabled) { @@ -774,11 +788,11 @@ public class Config { } /** - * @deprecated This method should only be used internally. Use the instance getter instead {@link - * #isJmxFetchIntegrationEnabled(SortedSet, boolean)}. * @param integrationNames * @param defaultEnabled * @return + * @deprecated This method should only be used internally. Use the instance getter instead {@link + * #isJmxFetchIntegrationEnabled(SortedSet, boolean)}. */ public static boolean jmxFetchIntegrationEnabled( final SortedSet integrationNames, final boolean defaultEnabled) { @@ -803,11 +817,11 @@ public class Config { } /** - * @deprecated This method should only be used internally. Use the instance getter instead {@link - * #isTraceAnalyticsIntegrationEnabled(SortedSet, boolean)}. * @param integrationNames * @param defaultEnabled * @return + * @deprecated This method should only be used internally. Use the instance getter instead {@link + * #isTraceAnalyticsIntegrationEnabled(SortedSet, boolean)}. */ public static boolean traceAnalyticsIntegrationEnabled( final SortedSet integrationNames, final boolean defaultEnabled) { diff --git a/dd-trace-api/src/test/groovy/datadog/trace/api/ConfigTest.groovy b/dd-trace-api/src/test/groovy/datadog/trace/api/ConfigTest.groovy index db37f07a6a..7c44765790 100644 --- a/dd-trace-api/src/test/groovy/datadog/trace/api/ConfigTest.groovy +++ b/dd-trace-api/src/test/groovy/datadog/trace/api/ConfigTest.groovy @@ -40,6 +40,7 @@ import static datadog.trace.api.Config.PROFILING_PROXY_HOST import static datadog.trace.api.Config.PROFILING_PROXY_PASSWORD import static datadog.trace.api.Config.PROFILING_PROXY_PORT import static datadog.trace.api.Config.PROFILING_PROXY_USERNAME +import static datadog.trace.api.Config.PROFILING_SITE import static datadog.trace.api.Config.PROFILING_START_DELAY import static datadog.trace.api.Config.PROFILING_START_FORCE_FIRST import static datadog.trace.api.Config.PROFILING_TAGS @@ -133,7 +134,8 @@ class ConfigTest extends DDSpecification { config.healthMetricsStatsdPort == null config.profilingEnabled == false - config.profilingUrl == Config.DEFAULT_PROFILING_URL + config.profilingSite == Config.DEFAULT_PROFILING_SITE + config.profilingUrl == null config.profilingApiKey == null config.mergedProfilingTags == [(HOST_TAG): config.getHostName(), (RUNTIME_ID_TAG): config.getRuntimeId(), (SERVICE_TAG): config.serviceName, (LANGUAGE_TAG_KEY): LANGUAGE_TAG_VALUE] config.profilingStartDelay == 10 @@ -198,6 +200,7 @@ class ConfigTest extends DDSpecification { prop.setProperty(TRACE_RATE_LIMIT, "200") prop.setProperty(PROFILING_ENABLED, "true") + prop.setProperty(PROFILING_SITE, "new site") prop.setProperty(PROFILING_URL, "new url") prop.setProperty(PROFILING_API_KEY, "new api key") prop.setProperty(PROFILING_TAGS, "f:6,host:test-host") @@ -254,6 +257,7 @@ class ConfigTest extends DDSpecification { config.traceRateLimit == 200 config.profilingEnabled == true + config.profilingSite == "new site" config.profilingUrl == "new url" config.profilingApiKey == "new api key" // we can still override via internal properties object config.mergedProfilingTags == [b: "2", f: "6", (HOST_TAG): "test-host", (RUNTIME_ID_TAG): config.getRuntimeId(), (SERVICE_TAG): config.serviceName, (LANGUAGE_TAG_KEY): LANGUAGE_TAG_VALUE] @@ -310,6 +314,7 @@ class ConfigTest extends DDSpecification { System.setProperty(PREFIX + TRACE_RATE_LIMIT, "200") System.setProperty(PREFIX + PROFILING_ENABLED, "true") + System.setProperty(PREFIX + PROFILING_SITE, "new site") System.setProperty(PREFIX + PROFILING_URL, "new url") System.setProperty(PREFIX + PROFILING_API_KEY, "new api key") System.setProperty(PREFIX + PROFILING_TAGS, "f:6,host:test-host") @@ -366,6 +371,7 @@ class ConfigTest extends DDSpecification { config.traceRateLimit == 200 config.profilingEnabled == true + config.profilingSite == "new site" config.profilingUrl == "new url" config.profilingApiKey == null // system properties cannot be used to provide a key config.mergedProfilingTags == [b: "2", f: "6", (HOST_TAG): "test-host", (RUNTIME_ID_TAG): config.getRuntimeId(), (SERVICE_TAG): config.serviceName, (LANGUAGE_TAG_KEY): LANGUAGE_TAG_VALUE] @@ -1105,4 +1111,29 @@ class ConfigTest extends DDSpecification { config.mergedProfilingTags == [a: "1", f: "6", (HOST_TAG): config.getHostName(), (RUNTIME_ID_TAG): config.getRuntimeId(), (SERVICE_TAG): config.serviceName, (LANGUAGE_TAG_KEY): LANGUAGE_TAG_VALUE] } + + def "custom datadog site"() { + setup: + def prop = new Properties() + prop.setProperty(PROFILING_SITE, "some.new.site") + + when: + Config config = Config.get(prop) + + then: + config.getFinalProfilingUrl() == "https://intake.profile.some.new.site/v1/input" + } + + def "custom profiling url override"() { + setup: + def prop = new Properties() + prop.setProperty(PROFILING_SITE, "some.new.site") + prop.setProperty(PROFILING_URL, "https://some.new.url/goes/here") + + when: + Config config = Config.get(prop) + + then: + config.getFinalProfilingUrl() == "https://some.new.url/goes/here" + } } From e1f2052e25574d9007207bd0c1e407d6419087b6 Mon Sep 17 00:00:00 2001 From: Nikolay Martynov Date: Thu, 2 Apr 2020 16:03:03 -0400 Subject: [PATCH 32/43] Minor clean up and add one more test --- .../datadog/trace/api/ConfigTest.groovy | 43 +++++++++++-------- 1 file changed, 26 insertions(+), 17 deletions(-) diff --git a/dd-trace-api/src/test/groovy/datadog/trace/api/ConfigTest.groovy b/dd-trace-api/src/test/groovy/datadog/trace/api/ConfigTest.groovy index 4eb3ef7ad3..853552b7a9 100644 --- a/dd-trace-api/src/test/groovy/datadog/trace/api/ConfigTest.groovy +++ b/dd-trace-api/src/test/groovy/datadog/trace/api/ConfigTest.groovy @@ -407,23 +407,6 @@ class ConfigTest extends DDSpecification { config.profilingApiKey == "test-api-key" } - def "sensitive information removed for toString/debug log"() { - setup: - environmentVariables.set(DD_PROFILING_API_KEY_ENV, "test-secret-api-key") - environmentVariables.set(DD_PROFILING_PROXY_PASSWORD_ENV, "test-secret-proxy-password") - - when: - def config = new Config() - - then: - config.toString().contains("profilingApiKey=****"); - !config.toString().contains("test-secret-api-key") - config.toString().contains("profilingProxyPassword=****"); - !config.toString().contains("test-secret-proxy-password") - config.getProfilingApiKey() == "test-secret-api-key" - config.getProfilingProxyPassword() == "test-secret-proxy-password" - } - def "sys props override env vars"() { setup: environmentVariables.set(DD_SERVICE_NAME_ENV, "still something else") @@ -1123,4 +1106,30 @@ class ConfigTest extends DDSpecification { config.mergedProfilingTags == [a: "1", f: "6", (HOST_TAG): config.getHostName(), (RUNTIME_ID_TAG): config.getRuntimeId(), (SERVICE_TAG): config.serviceName, (LANGUAGE_TAG_KEY): LANGUAGE_TAG_VALUE] } + + def "sensitive information removed for toString/debug log"() { + setup: + environmentVariables.set(DD_PROFILING_API_KEY_ENV, "test-secret-api-key") + environmentVariables.set(DD_PROFILING_PROXY_PASSWORD_ENV, "test-secret-proxy-password") + + when: + def config = new Config() + + then: + config.toString().contains("profilingApiKey=****") + !config.toString().contains("test-secret-api-key") + config.toString().contains("profilingProxyPassword=****") + !config.toString().contains("test-secret-proxy-password") + config.profilingApiKey == "test-secret-api-key" + config.profilingProxyPassword == "test-secret-proxy-password" + } + + def "toString works when passwords are empty"() { + when: + def config = new Config() + + then: + config.toString().contains("profilingApiKey=null") + config.toString().contains("profilingProxyPassword=null") + } } From e812d327fb0845c1add520f59d0d745eb8a362e0 Mon Sep 17 00:00:00 2001 From: Nikolay Martynov Date: Thu, 2 Apr 2020 16:20:41 -0400 Subject: [PATCH 33/43] Rename DD_PROFILING_SITE to DD_SITE --- .../src/main/java/datadog/trace/api/Config.java | 17 +++++++++++------ .../groovy/datadog/trace/api/ConfigTest.groovy | 16 ++++++++-------- 2 files changed, 19 insertions(+), 14 deletions(-) diff --git a/dd-trace-api/src/main/java/datadog/trace/api/Config.java b/dd-trace-api/src/main/java/datadog/trace/api/Config.java index b645cb5fa8..7330ef78c6 100644 --- a/dd-trace-api/src/main/java/datadog/trace/api/Config.java +++ b/dd-trace-api/src/main/java/datadog/trace/api/Config.java @@ -49,6 +49,7 @@ public class Config { private static final Pattern ENV_REPLACEMENT = Pattern.compile("[^a-zA-Z0-9_]"); public static final String CONFIGURATION_FILE = "trace.config"; + public static final String SITE = "site"; public static final String SERVICE_NAME = "service.name"; public static final String TRACE_ENABLED = "trace.enabled"; public static final String INTEGRATIONS_ENABLED = "integrations.enabled"; @@ -109,7 +110,6 @@ public class Config { public static final String LOGS_INJECTION_ENABLED = "logs.injection"; public static final String PROFILING_ENABLED = "profiling.enabled"; - public static final String PROFILING_SITE = "profiling.site"; public static final String PROFILING_URL = "profiling.url"; public static final String PROFILING_API_KEY = "profiling.api-key"; @@ -138,6 +138,7 @@ public class Config { public static final String LANGUAGE_TAG_KEY = "language"; public static final String LANGUAGE_TAG_VALUE = "jvm"; + public static final String DEFAULT_SITE = "datadoghq.com"; public static final String DEFAULT_SERVICE_NAME = "unnamed-java-app"; private static final boolean DEFAULT_TRACE_ENABLED = true; @@ -177,7 +178,6 @@ public class Config { public static final boolean DEFAULT_LOGS_INJECTION_ENABLED = false; public static final boolean DEFAULT_PROFILING_ENABLED = false; - public static final String DEFAULT_PROFILING_SITE = "datadoghq.com"; public static final int DEFAULT_PROFILING_START_DELAY = 10; public static final boolean DEFAULT_PROFILING_START_FORCE_FIRST = false; public static final int DEFAULT_PROFILING_UPLOAD_PERIOD = 60; // 1 min @@ -211,6 +211,12 @@ public class Config { */ @Getter private final String runtimeId; + /** + * Note: this has effect only on profiling site. Traces are sent to Datadog agent and are not + * affected by this setting. + */ + @Getter private final String site; + @Getter private final String serviceName; @Getter private final boolean traceEnabled; @Getter private final boolean integrationsEnabled; @@ -272,7 +278,6 @@ public class Config { @Getter private final Double traceRateLimit; @Getter private final boolean profilingEnabled; - @Getter private final String profilingSite; private final String profilingUrl; @Getter private final String profilingApiKey; private final Map profilingTags; @@ -297,6 +302,7 @@ public class Config { runtimeId = UUID.randomUUID().toString(); + site = getSettingFromEnvironment(SITE, DEFAULT_SITE); serviceName = getSettingFromEnvironment(SERVICE_NAME, DEFAULT_SERVICE_NAME); traceEnabled = getBooleanSettingFromEnvironment(TRACE_ENABLED, DEFAULT_TRACE_ENABLED); @@ -419,7 +425,6 @@ public class Config { profilingEnabled = getBooleanSettingFromEnvironment(PROFILING_ENABLED, DEFAULT_PROFILING_ENABLED); - profilingSite = getSettingFromEnvironment(PROFILING_SITE, DEFAULT_PROFILING_SITE); profilingUrl = getSettingFromEnvironment(PROFILING_URL, null); // Note: We do not want APiKey to be loaded from property for security reasons // Note: we do not use defined default here @@ -483,6 +488,7 @@ public class Config { private Config(final Properties properties, final Config parent) { runtimeId = parent.runtimeId; + site = properties.getProperty(SITE, parent.site); serviceName = properties.getProperty(SERVICE_NAME, parent.serviceName); traceEnabled = getPropertyBooleanValue(properties, TRACE_ENABLED, parent.traceEnabled); @@ -614,7 +620,6 @@ public class Config { profilingEnabled = getPropertyBooleanValue(properties, PROFILING_ENABLED, parent.profilingEnabled); - profilingSite = properties.getProperty(PROFILING_SITE, parent.profilingSite); profilingUrl = properties.getProperty(PROFILING_URL, parent.profilingUrl); profilingApiKey = properties.getProperty(PROFILING_API_KEY, parent.profilingApiKey); profilingTags = getPropertyMapValue(properties, PROFILING_TAGS, parent.profilingTags); @@ -742,7 +747,7 @@ public class Config { public String getFinalProfilingUrl() { if (profilingUrl == null) { - return String.format(PROFILING_URL_TEMPLATE, profilingSite); + return String.format(PROFILING_URL_TEMPLATE, site); } else { return profilingUrl; } diff --git a/dd-trace-api/src/test/groovy/datadog/trace/api/ConfigTest.groovy b/dd-trace-api/src/test/groovy/datadog/trace/api/ConfigTest.groovy index 7c44765790..9fc303ac30 100644 --- a/dd-trace-api/src/test/groovy/datadog/trace/api/ConfigTest.groovy +++ b/dd-trace-api/src/test/groovy/datadog/trace/api/ConfigTest.groovy @@ -40,7 +40,6 @@ import static datadog.trace.api.Config.PROFILING_PROXY_HOST import static datadog.trace.api.Config.PROFILING_PROXY_PASSWORD import static datadog.trace.api.Config.PROFILING_PROXY_PORT import static datadog.trace.api.Config.PROFILING_PROXY_USERNAME -import static datadog.trace.api.Config.PROFILING_SITE import static datadog.trace.api.Config.PROFILING_START_DELAY import static datadog.trace.api.Config.PROFILING_START_FORCE_FIRST import static datadog.trace.api.Config.PROFILING_TAGS @@ -56,6 +55,7 @@ import static datadog.trace.api.Config.RUNTIME_ID_TAG import static datadog.trace.api.Config.SERVICE_MAPPING import static datadog.trace.api.Config.SERVICE_NAME import static datadog.trace.api.Config.SERVICE_TAG +import static datadog.trace.api.Config.SITE import static datadog.trace.api.Config.SPAN_TAGS import static datadog.trace.api.Config.SPLIT_BY_TAGS import static datadog.trace.api.Config.TAGS @@ -134,7 +134,7 @@ class ConfigTest extends DDSpecification { config.healthMetricsStatsdPort == null config.profilingEnabled == false - config.profilingSite == Config.DEFAULT_PROFILING_SITE + config.site == Config.DEFAULT_SITE config.profilingUrl == null config.profilingApiKey == null config.mergedProfilingTags == [(HOST_TAG): config.getHostName(), (RUNTIME_ID_TAG): config.getRuntimeId(), (SERVICE_TAG): config.serviceName, (LANGUAGE_TAG_KEY): LANGUAGE_TAG_VALUE] @@ -200,7 +200,7 @@ class ConfigTest extends DDSpecification { prop.setProperty(TRACE_RATE_LIMIT, "200") prop.setProperty(PROFILING_ENABLED, "true") - prop.setProperty(PROFILING_SITE, "new site") + prop.setProperty(SITE, "new site") prop.setProperty(PROFILING_URL, "new url") prop.setProperty(PROFILING_API_KEY, "new api key") prop.setProperty(PROFILING_TAGS, "f:6,host:test-host") @@ -257,7 +257,7 @@ class ConfigTest extends DDSpecification { config.traceRateLimit == 200 config.profilingEnabled == true - config.profilingSite == "new site" + config.site == "new site" config.profilingUrl == "new url" config.profilingApiKey == "new api key" // we can still override via internal properties object config.mergedProfilingTags == [b: "2", f: "6", (HOST_TAG): "test-host", (RUNTIME_ID_TAG): config.getRuntimeId(), (SERVICE_TAG): config.serviceName, (LANGUAGE_TAG_KEY): LANGUAGE_TAG_VALUE] @@ -314,7 +314,7 @@ class ConfigTest extends DDSpecification { System.setProperty(PREFIX + TRACE_RATE_LIMIT, "200") System.setProperty(PREFIX + PROFILING_ENABLED, "true") - System.setProperty(PREFIX + PROFILING_SITE, "new site") + System.setProperty(PREFIX + SITE, "new site") System.setProperty(PREFIX + PROFILING_URL, "new url") System.setProperty(PREFIX + PROFILING_API_KEY, "new api key") System.setProperty(PREFIX + PROFILING_TAGS, "f:6,host:test-host") @@ -371,7 +371,7 @@ class ConfigTest extends DDSpecification { config.traceRateLimit == 200 config.profilingEnabled == true - config.profilingSite == "new site" + config.site == "new site" config.profilingUrl == "new url" config.profilingApiKey == null // system properties cannot be used to provide a key config.mergedProfilingTags == [b: "2", f: "6", (HOST_TAG): "test-host", (RUNTIME_ID_TAG): config.getRuntimeId(), (SERVICE_TAG): config.serviceName, (LANGUAGE_TAG_KEY): LANGUAGE_TAG_VALUE] @@ -1115,7 +1115,7 @@ class ConfigTest extends DDSpecification { def "custom datadog site"() { setup: def prop = new Properties() - prop.setProperty(PROFILING_SITE, "some.new.site") + prop.setProperty(SITE, "some.new.site") when: Config config = Config.get(prop) @@ -1127,7 +1127,7 @@ class ConfigTest extends DDSpecification { def "custom profiling url override"() { setup: def prop = new Properties() - prop.setProperty(PROFILING_SITE, "some.new.site") + prop.setProperty(SITE, "some.new.site") prop.setProperty(PROFILING_URL, "https://some.new.url/goes/here") when: From b429cf4c52de5a45a33fc147f1acec8f26f5917b Mon Sep 17 00:00:00 2001 From: Nikolay Martynov Date: Fri, 3 Apr 2020 07:52:26 -0400 Subject: [PATCH 34/43] Add some deprecation notes --- dd-trace-api/src/main/java/datadog/trace/api/Config.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dd-trace-api/src/main/java/datadog/trace/api/Config.java b/dd-trace-api/src/main/java/datadog/trace/api/Config.java index 7330ef78c6..bc5b95124b 100644 --- a/dd-trace-api/src/main/java/datadog/trace/api/Config.java +++ b/dd-trace-api/src/main/java/datadog/trace/api/Config.java @@ -97,7 +97,7 @@ public class Config { public static final String JMX_FETCH_ENABLED = "jmxfetch.enabled"; public static final String JMX_FETCH_CONFIG_DIR = "jmxfetch.config.dir"; public static final String JMX_FETCH_CONFIG = "jmxfetch.config"; - public static final String JMX_FETCH_METRICS_CONFIGS = "jmxfetch.metrics-configs"; + @Deprecated public static final String JMX_FETCH_METRICS_CONFIGS = "jmxfetch.metrics-configs"; public static final String JMX_FETCH_CHECK_PERIOD = "jmxfetch.check-period"; public static final String JMX_FETCH_REFRESH_BEANS_PERIOD = "jmxfetch.refresh-beans-period"; public static final String JMX_FETCH_STATSD_HOST = "jmxfetch.statsd.host"; @@ -110,8 +110,8 @@ public class Config { public static final String LOGS_INJECTION_ENABLED = "logs.injection"; public static final String PROFILING_ENABLED = "profiling.enabled"; + @Deprecated // Use dd.site instead public static final String PROFILING_URL = "profiling.url"; - public static final String PROFILING_API_KEY = "profiling.api-key"; public static final String PROFILING_API_KEY_FILE = "profiling.api-key-file"; public static final String PROFILING_API_KEY_OLD = "profiling.apikey"; @@ -278,7 +278,7 @@ public class Config { @Getter private final Double traceRateLimit; @Getter private final boolean profilingEnabled; - private final String profilingUrl; + @Deprecated private final String profilingUrl; @Getter private final String profilingApiKey; private final Map profilingTags; @Getter private final int profilingStartDelay; From 343a77d6190d687a97c1d45f4c73a1540af0e85a Mon Sep 17 00:00:00 2001 From: jean-philippe bempel Date: Fri, 3 Apr 2020 19:59:36 +0200 Subject: [PATCH 35/43] Add fix suggestion when JFR API not found suggests to use zulu 1.8u212 at least which include JFR or OpenJDK11+ --- .../controller/ControllerFactory.java | 24 +++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/dd-java-agent/agent-profiling/profiling-controller/src/main/java/com/datadog/profiling/controller/ControllerFactory.java b/dd-java-agent/agent-profiling/profiling-controller/src/main/java/com/datadog/profiling/controller/ControllerFactory.java index a1991a5d2d..923727cb14 100644 --- a/dd-java-agent/agent-profiling/profiling-controller/src/main/java/com/datadog/profiling/controller/ControllerFactory.java +++ b/dd-java-agent/agent-profiling/profiling-controller/src/main/java/com/datadog/profiling/controller/ControllerFactory.java @@ -49,8 +49,28 @@ public final class ControllerFactory { | InstantiationException | IllegalAccessException | InvocationTargetException e) { - throw new UnsupportedEnvironmentException( - "The JFR controller could not find a supported JFR API", e); + String exMsg = "The JFR controller could not find a supported JFR API" + + getFixProposalMessage(); + throw new UnsupportedEnvironmentException(exMsg, e); + } + } + + private static String getFixProposalMessage() { + try { + String javaVersion = System.getProperty("java.version"); + if (javaVersion == null) { + return ""; + } + String javaVendor = System.getProperty("java.vendor", ""); + if (javaVersion.startsWith("1.8")) { + if (javaVendor.startsWith("Azul Systems")) { + return ", use Azul zulu version 1.8.0_212+"; + } + // TODO Add version minimum once JFR backported into OpenJDK distros + } + return ", use OpenJDK 11+ or Azul zulu version 1.8.0_212+"; + } catch (Exception ex) { + return ""; } } } From ce26d0532e781354f69225efb2b509b66e587abd Mon Sep 17 00:00:00 2001 From: jean-philippe bempel Date: Mon, 6 Apr 2020 12:19:20 +0200 Subject: [PATCH 36/43] Add test for improved messages --- .../controller/ControllerFactoryTest.java | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/dd-java-agent/agent-profiling/profiling-controller/src/test/java/com/datadog/profiling/controller/ControllerFactoryTest.java b/dd-java-agent/agent-profiling/profiling-controller/src/test/java/com/datadog/profiling/controller/ControllerFactoryTest.java index 323bbe6fe2..1bfe373160 100644 --- a/dd-java-agent/agent-profiling/profiling-controller/src/test/java/com/datadog/profiling/controller/ControllerFactoryTest.java +++ b/dd-java-agent/agent-profiling/profiling-controller/src/test/java/com/datadog/profiling/controller/ControllerFactoryTest.java @@ -1,5 +1,6 @@ package com.datadog.profiling.controller; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.condition.JRE.JAVA_8; @@ -19,10 +20,18 @@ public class ControllerFactoryTest { @Test @EnabledOnJre({JAVA_8}) public void testCreateControllerJava8() { - assertThrows( - UnsupportedEnvironmentException.class, - () -> { - ControllerFactory.createController(config); - }); + System.out.println("Java Version: " + System.getProperty("java.version") + " " + System.getProperty("java.vendor")); + UnsupportedEnvironmentException unsupportedEnvironmentException = assertThrows( + UnsupportedEnvironmentException.class, + () -> { + ControllerFactory.createController(config); + }); + System.out.println(unsupportedEnvironmentException.getMessage()); + System.out.println(System.getProperty("java.vendor")); + String expected = "The JFR controller could not find a supported JFR API, use OpenJDK 11+ or Azul zulu version 1.8.0_212+"; + if ("Azul Systems, Inc.".equals(System.getProperty("java.vendor"))) { + expected = "The JFR controller could not find a supported JFR API, use Azul zulu version 1.8.0_212+"; + } + assertEquals(expected, unsupportedEnvironmentException.getMessage()); } } From 658544fb38843b543d6a6be8ed7f755386b0b08d Mon Sep 17 00:00:00 2001 From: jean-philippe bempel Date: Mon, 6 Apr 2020 13:26:55 +0200 Subject: [PATCH 37/43] Fix formatting --- .../controller/ControllerFactory.java | 4 ++-- .../controller/ControllerFactoryTest.java | 20 +++++++++---------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/dd-java-agent/agent-profiling/profiling-controller/src/main/java/com/datadog/profiling/controller/ControllerFactory.java b/dd-java-agent/agent-profiling/profiling-controller/src/main/java/com/datadog/profiling/controller/ControllerFactory.java index 923727cb14..a9cd29e5be 100644 --- a/dd-java-agent/agent-profiling/profiling-controller/src/main/java/com/datadog/profiling/controller/ControllerFactory.java +++ b/dd-java-agent/agent-profiling/profiling-controller/src/main/java/com/datadog/profiling/controller/ControllerFactory.java @@ -49,8 +49,8 @@ public final class ControllerFactory { | InstantiationException | IllegalAccessException | InvocationTargetException e) { - String exMsg = "The JFR controller could not find a supported JFR API" - + getFixProposalMessage(); + String exMsg = + "The JFR controller could not find a supported JFR API" + getFixProposalMessage(); throw new UnsupportedEnvironmentException(exMsg, e); } } diff --git a/dd-java-agent/agent-profiling/profiling-controller/src/test/java/com/datadog/profiling/controller/ControllerFactoryTest.java b/dd-java-agent/agent-profiling/profiling-controller/src/test/java/com/datadog/profiling/controller/ControllerFactoryTest.java index 1bfe373160..972ee08a4a 100644 --- a/dd-java-agent/agent-profiling/profiling-controller/src/test/java/com/datadog/profiling/controller/ControllerFactoryTest.java +++ b/dd-java-agent/agent-profiling/profiling-controller/src/test/java/com/datadog/profiling/controller/ControllerFactoryTest.java @@ -20,17 +20,17 @@ public class ControllerFactoryTest { @Test @EnabledOnJre({JAVA_8}) public void testCreateControllerJava8() { - System.out.println("Java Version: " + System.getProperty("java.version") + " " + System.getProperty("java.vendor")); - UnsupportedEnvironmentException unsupportedEnvironmentException = assertThrows( - UnsupportedEnvironmentException.class, - () -> { - ControllerFactory.createController(config); - }); - System.out.println(unsupportedEnvironmentException.getMessage()); - System.out.println(System.getProperty("java.vendor")); - String expected = "The JFR controller could not find a supported JFR API, use OpenJDK 11+ or Azul zulu version 1.8.0_212+"; + UnsupportedEnvironmentException unsupportedEnvironmentException = + assertThrows( + UnsupportedEnvironmentException.class, + () -> { + ControllerFactory.createController(config); + }); + String expected = + "The JFR controller could not find a supported JFR API, use OpenJDK 11+ or Azul zulu version 1.8.0_212+"; if ("Azul Systems, Inc.".equals(System.getProperty("java.vendor"))) { - expected = "The JFR controller could not find a supported JFR API, use Azul zulu version 1.8.0_212+"; + expected = + "The JFR controller could not find a supported JFR API, use Azul zulu version 1.8.0_212+"; } assertEquals(expected, unsupportedEnvironmentException.getMessage()); } From 2870819fa6fc162f05a9f3831131f1fea184abec Mon Sep 17 00:00:00 2001 From: Nikolay Martynov Date: Mon, 6 Apr 2020 08:08:03 -0400 Subject: [PATCH 38/43] Update gradle to 6.3 --- gradle/wrapper/gradle-wrapper.properties | 2 +- gradlew.bat | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 84a906615c..6623300beb 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-6.2.2-all.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-6.3-all.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/gradlew.bat b/gradlew.bat index 24467a141f..9109989e3c 100644 --- a/gradlew.bat +++ b/gradlew.bat @@ -29,6 +29,9 @@ if "%DIRNAME%" == "" set DIRNAME=. set APP_BASE_NAME=%~n0 set APP_HOME=%DIRNAME% +@rem Resolve any "." and ".." in APP_HOME to make it shorter. +for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi + @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" From 309f9e2ddb1f21df5bb954844c9b4abc92566dbb Mon Sep 17 00:00:00 2001 From: Nikolay Martynov Date: Mon, 6 Apr 2020 11:44:27 -0400 Subject: [PATCH 39/43] Introduce to API key config option To stream line configuration in the future intropduce `DD_API_KEY`/`DD_API_KEY_FILE` to provide agent with API key. Currently this is only used for profiling but later may be used for other parts of the agent too. --- .../profiling/uploader/RecordingUploader.java | 2 +- .../uploader/RecordingUploaderTest.java | 10 +- .../profiling/agent/ProfilingAgent.java | 3 +- .../trace/agent/CustomLogManagerTest.groovy | 14 +-- ...ngIntegrationContinuousProfilesTest.groovy | 4 +- .../smoketest/AbstractSmokeTest.groovy | 4 +- .../main/java/datadog/trace/api/Config.java | 84 ++++++++----- .../datadog/trace/api/ConfigTest.groovy | 111 +++++++++++++----- .../src/test/resources/apikey.very-old | 1 + 9 files changed, 157 insertions(+), 76 deletions(-) create mode 100644 dd-trace-api/src/test/resources/apikey.very-old diff --git a/dd-java-agent/agent-profiling/profiling-uploader/src/main/java/com/datadog/profiling/uploader/RecordingUploader.java b/dd-java-agent/agent-profiling/profiling-uploader/src/main/java/com/datadog/profiling/uploader/RecordingUploader.java index 5be5bf3974..761f0316f1 100644 --- a/dd-java-agent/agent-profiling/profiling-uploader/src/main/java/com/datadog/profiling/uploader/RecordingUploader.java +++ b/dd-java-agent/agent-profiling/profiling-uploader/src/main/java/com/datadog/profiling/uploader/RecordingUploader.java @@ -122,7 +122,7 @@ public final class RecordingUploader { public RecordingUploader(final Config config) { url = config.getFinalProfilingUrl(); - apiKey = config.getProfilingApiKey(); + apiKey = config.getApiKey(); /* FIXME: currently `Config` class cannot get access to some pieces of information we need here: diff --git a/dd-java-agent/agent-profiling/profiling-uploader/src/test/java/com/datadog/profiling/uploader/RecordingUploaderTest.java b/dd-java-agent/agent-profiling/profiling-uploader/src/test/java/com/datadog/profiling/uploader/RecordingUploaderTest.java index 77b492e395..c40e92b7a8 100644 --- a/dd-java-agent/agent-profiling/profiling-uploader/src/test/java/com/datadog/profiling/uploader/RecordingUploaderTest.java +++ b/dd-java-agent/agent-profiling/profiling-uploader/src/test/java/com/datadog/profiling/uploader/RecordingUploaderTest.java @@ -68,7 +68,7 @@ import org.mockito.junit.jupiter.MockitoExtension; @ExtendWith(MockitoExtension.class) public class RecordingUploaderTest { - private static final String APIKEY_VALUE = "testkey"; + private static final String API_KEY_VALUE = "testkey"; private static final String URL_PATH = "/lalala"; private static final String RECORDING_RESOURCE = "test-recording.jfr"; private static final String RECODING_NAME_PREFIX = "test-recording-"; @@ -121,7 +121,7 @@ public class RecordingUploaderTest { url = server.url(URL_PATH); when(config.getFinalProfilingUrl()).thenReturn(server.url(URL_PATH).toString()); - when(config.getProfilingApiKey()).thenReturn(APIKEY_VALUE); + when(config.getApiKey()).thenReturn(API_KEY_VALUE); when(config.getMergedProfilingTags()).thenReturn(TAGS); when(config.getProfilingUploadTimeout()).thenReturn((int) REQUEST_TIMEOUT.getSeconds()); @@ -152,7 +152,7 @@ public class RecordingUploaderTest { final RecordedRequest recordedRequest = server.takeRequest(5, TimeUnit.SECONDS); assertEquals(url, recordedRequest.getRequestUrl()); - assertEquals(APIKEY_VALUE, recordedRequest.getHeader("DD-API-KEY")); + assertEquals(API_KEY_VALUE, recordedRequest.getHeader("DD-API-KEY")); final Multimap parameters = ProfilingTestUtils.parseProfilingRequestParameters(recordedRequest); @@ -215,7 +215,7 @@ public class RecordingUploaderTest { final RecordedRequest recordedFirstRequest = server.takeRequest(5, TimeUnit.SECONDS); assertEquals(server.url(""), recordedFirstRequest.getRequestUrl()); - assertEquals(APIKEY_VALUE, recordedFirstRequest.getHeader("DD-API-KEY")); + assertEquals(API_KEY_VALUE, recordedFirstRequest.getHeader("DD-API-KEY")); assertNull(recordedFirstRequest.getHeader("Proxy-Authorization")); assertEquals(backendHost, recordedFirstRequest.getHeader("Host")); assertEquals( @@ -223,7 +223,7 @@ public class RecordingUploaderTest { final RecordedRequest recordedSecondRequest = server.takeRequest(5, TimeUnit.SECONDS); assertEquals(server.url(""), recordedSecondRequest.getRequestUrl()); - assertEquals(APIKEY_VALUE, recordedSecondRequest.getHeader("DD-API-KEY")); + assertEquals(API_KEY_VALUE, recordedSecondRequest.getHeader("DD-API-KEY")); assertEquals( Credentials.basic("username", "password"), recordedSecondRequest.getHeader("Proxy-Authorization")); diff --git a/dd-java-agent/agent-profiling/src/main/java/com/datadog/profiling/agent/ProfilingAgent.java b/dd-java-agent/agent-profiling/src/main/java/com/datadog/profiling/agent/ProfilingAgent.java index 3d817b2bf7..0e7165253d 100644 --- a/dd-java-agent/agent-profiling/src/main/java/com/datadog/profiling/agent/ProfilingAgent.java +++ b/dd-java-agent/agent-profiling/src/main/java/com/datadog/profiling/agent/ProfilingAgent.java @@ -34,7 +34,7 @@ public class ProfilingAgent { log.info("Profiling: disabled"); return; } - if (config.getProfilingApiKey() == null) { + if (config.getApiKey() == null) { log.info("Profiling: no API key, profiling disabled"); return; } @@ -80,6 +80,7 @@ public class ProfilingAgent { } private static class ShutdownHook extends Thread { + private final WeakReference profilerRef; private final WeakReference uploaderRef; diff --git a/dd-java-agent/src/test/groovy/datadog/trace/agent/CustomLogManagerTest.groovy b/dd-java-agent/src/test/groovy/datadog/trace/agent/CustomLogManagerTest.groovy index 8bf9e25266..ab185252e1 100644 --- a/dd-java-agent/src/test/groovy/datadog/trace/agent/CustomLogManagerTest.groovy +++ b/dd-java-agent/src/test/groovy/datadog/trace/agent/CustomLogManagerTest.groovy @@ -12,7 +12,7 @@ import spock.lang.Timeout class CustomLogManagerTest extends Specification { private static final String DEFAULT_LOG_LEVEL = "debug" - private static final String PROFILING_API_KEY = "some-api-key" + private static final String API_KEY = "some-api-key" // Run all tests using forked jvm because groovy has already set the global log manager def "agent services starts up in premain with no custom log manager set"() { @@ -20,7 +20,7 @@ class CustomLogManagerTest extends Specification { IntegrationTestUtils.runOnSeparateJvm(LogManagerSetter.getName() , ["-Ddd.jmxfetch.enabled=true", "-Ddd.jmxfetch.refresh-beans-period=1", "-Ddd.profiling.enabled=true", "-Ddatadog.slf4j.simpleLogger.defaultLogLevel=$DEFAULT_LOG_LEVEL"] as String[] , "" as String[] - , ["DD_PROFILING_APIKEY": PROFILING_API_KEY] + , ["DD_API_KEY": API_KEY] , true) == 0 } @@ -29,7 +29,7 @@ class CustomLogManagerTest extends Specification { IntegrationTestUtils.runOnSeparateJvm(LogManagerSetter.getName() , ["-Ddd.jmxfetch.enabled=true", "-Ddd.jmxfetch.refresh-beans-period=1", "-Ddd.profiling.enabled=true", "-Ddatadog.slf4j.simpleLogger.defaultLogLevel=$DEFAULT_LOG_LEVEL", "-Djava.util.logging.manager=jvmbootstraptest.CustomLogManager"] as String[] , "" as String[] - , ["DD_PROFILING_APIKEY": PROFILING_API_KEY] + , ["DD_API_KEY": API_KEY] , true) == 0 } @@ -38,7 +38,7 @@ class CustomLogManagerTest extends Specification { IntegrationTestUtils.runOnSeparateJvm(LogManagerSetter.getName() , ["-Ddd.jmxfetch.enabled=true", "-Ddd.jmxfetch.refresh-beans-period=1", "-Ddd.profiling.enabled=true", "-Ddatadog.slf4j.simpleLogger.defaultLogLevel=$DEFAULT_LOG_LEVEL", "-Djava.util.logging.manager=jvmbootstraptest.MissingLogManager"] as String[] , "" as String[] - , ["DD_PROFILING_APIKEY": PROFILING_API_KEY] + , ["DD_API_KEY": API_KEY] , true) == 0 } @@ -47,7 +47,7 @@ class CustomLogManagerTest extends Specification { IntegrationTestUtils.runOnSeparateJvm(LogManagerSetter.getName() , ["-Ddd.jmxfetch.enabled=true", "-Ddd.jmxfetch.refresh-beans-period=1", "-Ddd.profiling.enabled=true", "-Ddatadog.slf4j.simpleLogger.defaultLogLevel=$DEFAULT_LOG_LEVEL", "-Ddd.app.customlogmanager=true"] as String[] , "" as String[] - , ["DD_PROFILING_APIKEY": PROFILING_API_KEY] + , ["DD_API_KEY": API_KEY] , true) == 0 } @@ -56,7 +56,7 @@ class CustomLogManagerTest extends Specification { IntegrationTestUtils.runOnSeparateJvm(LogManagerSetter.getName() , ["-Ddd.jmxfetch.enabled=true", "-Ddd.jmxfetch.refresh-beans-period=1", "-Ddd.profiling.enabled=true", "-Ddatadog.slf4j.simpleLogger.defaultLogLevel=$DEFAULT_LOG_LEVEL"] as String[] , "" as String[] - , ["JBOSS_HOME": "/", "DD_PROFILING_APIKEY": PROFILING_API_KEY] + , ["JBOSS_HOME": "/", "DD_API_KEY": API_KEY] , true) == 0 } @@ -65,7 +65,7 @@ class CustomLogManagerTest extends Specification { IntegrationTestUtils.runOnSeparateJvm(LogManagerSetter.getName() , ["-Ddd.jmxfetch.enabled=true", "-Ddd.jmxfetch.refresh-beans-period=1", "-Ddd.profiling.enabled=true", "-Ddatadog.slf4j.simpleLogger.defaultLogLevel=$DEFAULT_LOG_LEVEL", "-Ddd.app.customlogmanager=false", "-Djava.util.logging.manager=jvmbootstraptest.CustomLogManager"] as String[] , "" as String[] - , ["JBOSS_HOME": "/", "DD_PROFILING_APIKEY": PROFILING_API_KEY] + , ["JBOSS_HOME": "/", "DD_API_KEY": API_KEY] , true) == 0 } } diff --git a/dd-smoke-tests/profiling-integration-tests/src/test/groovy/datadog/smoketest/ProfilingIntegrationContinuousProfilesTest.groovy b/dd-smoke-tests/profiling-integration-tests/src/test/groovy/datadog/smoketest/ProfilingIntegrationContinuousProfilesTest.groovy index 365e10f2ca..a76b2ee42f 100644 --- a/dd-smoke-tests/profiling-integration-tests/src/test/groovy/datadog/smoketest/ProfilingIntegrationContinuousProfilesTest.groovy +++ b/dd-smoke-tests/profiling-integration-tests/src/test/groovy/datadog/smoketest/ProfilingIntegrationContinuousProfilesTest.groovy @@ -57,7 +57,7 @@ class ProfilingIntegrationContinuousProfilesTest extends AbstractSmokeTest { then: firstRequest.getRequestUrl().toString() == profilingUrl - firstRequest.getHeader("DD-API-KEY") == PROFILING_API_KEY + firstRequest.getHeader("DD-API-KEY") == API_KEY firstRequestParameters.get("recording-name").get(0) == 'dd-profiling' firstRequestParameters.get("format").get(0) == "jfr" @@ -87,7 +87,7 @@ class ProfilingIntegrationContinuousProfilesTest extends AbstractSmokeTest { then: secondRequest.getRequestUrl().toString() == profilingUrl - secondRequest.getHeader("DD-API-KEY") == PROFILING_API_KEY + secondRequest.getHeader("DD-API-KEY") == API_KEY secondRequestParameters.get("recording-name").get(0) == 'dd-profiling' def secondStartTime = Instant.parse(secondRequestParameters.get("recording-start").get(0)) diff --git a/dd-smoke-tests/src/main/groovy/datadog/smoketest/AbstractSmokeTest.groovy b/dd-smoke-tests/src/main/groovy/datadog/smoketest/AbstractSmokeTest.groovy index 13f949c568..ff6b2bc83d 100644 --- a/dd-smoke-tests/src/main/groovy/datadog/smoketest/AbstractSmokeTest.groovy +++ b/dd-smoke-tests/src/main/groovy/datadog/smoketest/AbstractSmokeTest.groovy @@ -6,7 +6,7 @@ import spock.lang.Specification abstract class AbstractSmokeTest extends Specification { - public static final PROFILING_API_KEY = "org2_api_key" + public static final API_KEY = "some-api-key" public static final PROFILING_START_DELAY_SECONDS = 1 public static final int PROFILING_RECORDING_UPLOAD_PERIOD_SECONDS = 5 @@ -49,7 +49,7 @@ abstract class AbstractSmokeTest extends Specification { ProcessBuilder processBuilder = createProcessBuilder() processBuilder.environment().put("JAVA_HOME", System.getProperty("java.home")) - processBuilder.environment().put("DD_PROFILING_APIKEY", PROFILING_API_KEY) + processBuilder.environment().put("DD_API_KEY", API_KEY) processBuilder.redirectErrorStream(true) File log = new File("${buildDirectory}/reports/testProcess.${this.getClass().getName()}.log") diff --git a/dd-trace-api/src/main/java/datadog/trace/api/Config.java b/dd-trace-api/src/main/java/datadog/trace/api/Config.java index d8004e9950..5a5e0b45cd 100644 --- a/dd-trace-api/src/main/java/datadog/trace/api/Config.java +++ b/dd-trace-api/src/main/java/datadog/trace/api/Config.java @@ -49,6 +49,8 @@ public class Config { private static final Pattern ENV_REPLACEMENT = Pattern.compile("[^a-zA-Z0-9_]"); public static final String CONFIGURATION_FILE = "trace.config"; + public static final String API_KEY = "api-key"; + public static final String API_KEY_FILE = "api-key-file"; public static final String SITE = "site"; public static final String SERVICE_NAME = "service.name"; public static final String TRACE_ENABLED = "trace.enabled"; @@ -112,10 +114,14 @@ public class Config { public static final String PROFILING_ENABLED = "profiling.enabled"; @Deprecated // Use dd.site instead public static final String PROFILING_URL = "profiling.url"; - public static final String PROFILING_API_KEY = "profiling.api-key"; - public static final String PROFILING_API_KEY_FILE = "profiling.api-key-file"; - public static final String PROFILING_API_KEY_OLD = "profiling.apikey"; - public static final String PROFILING_API_KEY_FILE_OLD = "profiling.apikey.file"; + @Deprecated // Use dd.api-key instead + public static final String PROFILING_API_KEY_OLD = "profiling.api-key"; + @Deprecated // Use dd.api-key-file instead + public static final String PROFILING_API_KEY_FILE_OLD = "profiling.api-key-file"; + @Deprecated // Use dd.api-key instead + public static final String PROFILING_API_KEY_VERY_OLD = "profiling.apikey"; + @Deprecated // Use dd.api-key-file instead + public static final String PROFILING_API_KEY_FILE_VERY_OLD = "profiling.apikey.file"; public static final String PROFILING_TAGS = "profiling.tags"; public static final String PROFILING_START_DELAY = "profiling.start-delay"; // DANGEROUS! May lead on sigsegv on JVMs before 14 @@ -206,9 +212,9 @@ public class Config { private static final String INTERNAL_HOST_NAME = "_dd.hostname"; /** Used for masking sensitive information when doing toString */ - @ToString.Include(name = "profilingApiKey") + @ToString.Include(name = "apiKey") private String profilingApiKeyMasker() { - return profilingApiKey != null ? "****" : null; + return apiKey != null ? "****" : null; } /** Used for masking sensitive information when doing toString */ @@ -223,6 +229,11 @@ public class Config { */ @Getter private final String runtimeId; + /** + * Note: this has effect only on profiling site. Traces are sent to Datadog agent and are not + * affected by this setting. + */ + @Getter private final String apiKey; /** * Note: this has effect only on profiling site. Traces are sent to Datadog agent and are not * affected by this setting. @@ -291,7 +302,6 @@ public class Config { @Getter private final boolean profilingEnabled; @Deprecated private final String profilingUrl; - @Getter private final String profilingApiKey; private final Map profilingTags; @Getter private final int profilingStartDelay; @Getter private final boolean profilingStartForceFirst; @@ -314,6 +324,19 @@ public class Config { runtimeId = UUID.randomUUID().toString(); + // Note: We do not want APiKey to be loaded from property for security reasons + // Note: we do not use defined default here + // FIXME: We should use better authentication mechanism + final String apiKeyFile = getSettingFromEnvironment(API_KEY_FILE, null); + String tmpApiKey = System.getenv(propertyNameToEnvironmentVariableName(API_KEY)); + if (apiKeyFile != null) { + try { + tmpApiKey = + new String(Files.readAllBytes(Paths.get(apiKeyFile)), StandardCharsets.UTF_8).trim(); + } catch (final IOException e) { + log.error("Cannot read API key from file {}, skipping", apiKeyFile, e); + } + } site = getSettingFromEnvironment(SITE, DEFAULT_SITE); serviceName = getSettingFromEnvironment(SERVICE_NAME, DEFAULT_SERVICE_NAME); @@ -438,38 +461,38 @@ public class Config { profilingEnabled = getBooleanSettingFromEnvironment(PROFILING_ENABLED, DEFAULT_PROFILING_ENABLED); profilingUrl = getSettingFromEnvironment(PROFILING_URL, null); - // Note: We do not want APiKey to be loaded from property for security reasons - // Note: we do not use defined default here - // FIXME: We should use better authentication mechanism - final String profilingApiKeyFile = getSettingFromEnvironment(PROFILING_API_KEY_FILE, null); - String tmpProfilingApiKey = - System.getenv(propertyNameToEnvironmentVariableName(PROFILING_API_KEY)); - if (profilingApiKeyFile != null) { - try { - tmpProfilingApiKey = - new String(Files.readAllBytes(Paths.get(profilingApiKeyFile)), StandardCharsets.UTF_8) - .trim(); - } catch (final IOException e) { - log.error("Cannot read API key from file {}, skipping", profilingApiKeyFile, e); - } - } - if (tmpProfilingApiKey == null) { + + if (tmpApiKey == null) { final String oldProfilingApiKeyFile = getSettingFromEnvironment(PROFILING_API_KEY_FILE_OLD, null); - tmpProfilingApiKey = - System.getenv(propertyNameToEnvironmentVariableName(PROFILING_API_KEY_OLD)); + tmpApiKey = System.getenv(propertyNameToEnvironmentVariableName(PROFILING_API_KEY_OLD)); if (oldProfilingApiKeyFile != null) { try { - tmpProfilingApiKey = + tmpApiKey = new String( Files.readAllBytes(Paths.get(oldProfilingApiKeyFile)), StandardCharsets.UTF_8) .trim(); } catch (final IOException e) { - log.error("Cannot read API key from file {}, skipping", profilingApiKeyFile, e); + log.error("Cannot read API key from file {}, skipping", oldProfilingApiKeyFile, e); + } + } + } + if (tmpApiKey == null) { + final String veryOldProfilingApiKeyFile = + getSettingFromEnvironment(PROFILING_API_KEY_FILE_VERY_OLD, null); + tmpApiKey = System.getenv(propertyNameToEnvironmentVariableName(PROFILING_API_KEY_VERY_OLD)); + if (veryOldProfilingApiKeyFile != null) { + try { + tmpApiKey = + new String( + Files.readAllBytes(Paths.get(veryOldProfilingApiKeyFile)), + StandardCharsets.UTF_8) + .trim(); + } catch (final IOException e) { + log.error("Cannot read API key from file {}, skipping", veryOldProfilingApiKeyFile, e); } } } - profilingApiKey = tmpProfilingApiKey; profilingTags = getMapSettingFromEnvironment(PROFILING_TAGS, null); profilingStartDelay = @@ -493,6 +516,9 @@ public class Config { profilingProxyUsername = getSettingFromEnvironment(PROFILING_PROXY_USERNAME, null); profilingProxyPassword = getSettingFromEnvironment(PROFILING_PROXY_PASSWORD, null); + // Setting this last because we have a few places where this can come from + apiKey = tmpApiKey; + log.debug("New instance: {}", this); } @@ -500,6 +526,7 @@ public class Config { private Config(final Properties properties, final Config parent) { runtimeId = parent.runtimeId; + apiKey = properties.getProperty(API_KEY, parent.apiKey); site = properties.getProperty(SITE, parent.site); serviceName = properties.getProperty(SERVICE_NAME, parent.serviceName); @@ -633,7 +660,6 @@ public class Config { profilingEnabled = getPropertyBooleanValue(properties, PROFILING_ENABLED, parent.profilingEnabled); profilingUrl = properties.getProperty(PROFILING_URL, parent.profilingUrl); - profilingApiKey = properties.getProperty(PROFILING_API_KEY, parent.profilingApiKey); profilingTags = getPropertyMapValue(properties, PROFILING_TAGS, parent.profilingTags); profilingStartDelay = getPropertyIntegerValue(properties, PROFILING_START_DELAY, parent.profilingStartDelay); diff --git a/dd-trace-api/src/test/groovy/datadog/trace/api/ConfigTest.groovy b/dd-trace-api/src/test/groovy/datadog/trace/api/ConfigTest.groovy index 99e14c79d2..5b1bdf15fd 100644 --- a/dd-trace-api/src/test/groovy/datadog/trace/api/ConfigTest.groovy +++ b/dd-trace-api/src/test/groovy/datadog/trace/api/ConfigTest.groovy @@ -8,6 +8,8 @@ import org.junit.contrib.java.lang.system.RestoreSystemProperties import static datadog.trace.api.Config.AGENT_HOST import static datadog.trace.api.Config.AGENT_PORT_LEGACY import static datadog.trace.api.Config.AGENT_UNIX_DOMAIN_SOCKET +import static datadog.trace.api.Config.API_KEY +import static datadog.trace.api.Config.API_KEY_FILE import static datadog.trace.api.Config.CONFIGURATION_FILE import static datadog.trace.api.Config.DB_CLIENT_HOST_SPLIT_BY_INSTANCE import static datadog.trace.api.Config.DEFAULT_JMX_FETCH_STATSD_PORT @@ -32,9 +34,8 @@ import static datadog.trace.api.Config.LANGUAGE_TAG_VALUE import static datadog.trace.api.Config.PARTIAL_FLUSH_MIN_SPANS import static datadog.trace.api.Config.PREFIX import static datadog.trace.api.Config.PRIORITY_SAMPLING -import static datadog.trace.api.Config.PROFILING_API_KEY -import static datadog.trace.api.Config.PROFILING_API_KEY_FILE import static datadog.trace.api.Config.PROFILING_API_KEY_FILE_OLD +import static datadog.trace.api.Config.PROFILING_API_KEY_FILE_VERY_OLD import static datadog.trace.api.Config.PROFILING_ENABLED import static datadog.trace.api.Config.PROFILING_PROXY_HOST import static datadog.trace.api.Config.PROFILING_PROXY_PASSWORD @@ -75,6 +76,7 @@ class ConfigTest extends DDSpecification { @Rule public final EnvironmentVariables environmentVariables = new EnvironmentVariables() + private static final DD_API_KEY_ENV = "DD_API_KEY" private static final DD_SERVICE_NAME_ENV = "DD_SERVICE_NAME" private static final DD_TRACE_ENABLED_ENV = "DD_TRACE_ENABLED" private static final DD_WRITER_TYPE_ENV = "DD_WRITER_TYPE" @@ -91,8 +93,8 @@ class ConfigTest extends DDSpecification { private static final DD_AGENT_PORT_LEGACY_ENV = "DD_AGENT_PORT" private static final DD_TRACE_REPORT_HOSTNAME = "DD_TRACE_REPORT_HOSTNAME" - private static final DD_PROFILING_API_KEY_ENV = "DD_PROFILING_API_KEY" - private static final DD_PROFILING_API_KEY_OLD_ENV = "DD_PROFILING_APIKEY" + private static final DD_PROFILING_API_KEY_OLD_ENV = "DD_PROFILING_API_KEY" + private static final DD_PROFILING_API_KEY_VERY_OLD_ENV = "DD_PROFILING_APIKEY" private static final DD_PROFILING_TAGS_ENV = "DD_PROFILING_TAGS" private static final DD_PROFILING_PROXY_PASSWORD_ENV = "DD_PROFILING_PROXY_PASSWORD" @@ -101,6 +103,8 @@ class ConfigTest extends DDSpecification { Config config = provider() then: + config.apiKey == null + config.site == Config.DEFAULT_SITE config.serviceName == "unnamed-java-app" config.traceEnabled == true config.writerType == "DDAgentWriter" @@ -135,9 +139,7 @@ class ConfigTest extends DDSpecification { config.healthMetricsStatsdPort == null config.profilingEnabled == false - config.site == Config.DEFAULT_SITE config.profilingUrl == null - config.profilingApiKey == null config.mergedProfilingTags == [(HOST_TAG): config.getHostName(), (RUNTIME_ID_TAG): config.getRuntimeId(), (SERVICE_TAG): config.serviceName, (LANGUAGE_TAG_KEY): LANGUAGE_TAG_VALUE] config.profilingStartDelay == 10 config.profilingStartForceFirst == false @@ -162,6 +164,8 @@ class ConfigTest extends DDSpecification { def "specify overrides via properties"() { setup: def prop = new Properties() + prop.setProperty(API_KEY, "new api key") + prop.setProperty(SITE, "new site") prop.setProperty(SERVICE_NAME, "something else") prop.setProperty(TRACE_ENABLED, "false") prop.setProperty(WRITER_TYPE, "LoggingWriter") @@ -201,9 +205,7 @@ class ConfigTest extends DDSpecification { prop.setProperty(TRACE_RATE_LIMIT, "200") prop.setProperty(PROFILING_ENABLED, "true") - prop.setProperty(SITE, "new site") prop.setProperty(PROFILING_URL, "new url") - prop.setProperty(PROFILING_API_KEY, "new api key") prop.setProperty(PROFILING_TAGS, "f:6,host:test-host") prop.setProperty(PROFILING_START_DELAY, "1111") prop.setProperty(PROFILING_START_FORCE_FIRST, "true") @@ -220,6 +222,8 @@ class ConfigTest extends DDSpecification { Config config = Config.get(prop) then: + config.apiKey == "new api key" // we can still override via internal properties object + config.site == "new site" config.serviceName == "something else" config.traceEnabled == false config.writerType == "LoggingWriter" @@ -258,9 +262,7 @@ class ConfigTest extends DDSpecification { config.traceRateLimit == 200 config.profilingEnabled == true - config.site == "new site" config.profilingUrl == "new url" - config.profilingApiKey == "new api key" // we can still override via internal properties object config.mergedProfilingTags == [b: "2", f: "6", (HOST_TAG): "test-host", (RUNTIME_ID_TAG): config.getRuntimeId(), (SERVICE_TAG): config.serviceName, (LANGUAGE_TAG_KEY): LANGUAGE_TAG_VALUE] config.profilingStartDelay == 1111 config.profilingStartForceFirst == true @@ -276,6 +278,8 @@ class ConfigTest extends DDSpecification { def "specify overrides via system properties"() { setup: + System.setProperty(PREFIX + API_KEY, "new api key") + System.setProperty(PREFIX + SITE, "new site") System.setProperty(PREFIX + SERVICE_NAME, "something else") System.setProperty(PREFIX + TRACE_ENABLED, "false") System.setProperty(PREFIX + WRITER_TYPE, "LoggingWriter") @@ -315,9 +319,7 @@ class ConfigTest extends DDSpecification { System.setProperty(PREFIX + TRACE_RATE_LIMIT, "200") System.setProperty(PREFIX + PROFILING_ENABLED, "true") - System.setProperty(PREFIX + SITE, "new site") System.setProperty(PREFIX + PROFILING_URL, "new url") - System.setProperty(PREFIX + PROFILING_API_KEY, "new api key") System.setProperty(PREFIX + PROFILING_TAGS, "f:6,host:test-host") System.setProperty(PREFIX + PROFILING_START_DELAY, "1111") System.setProperty(PREFIX + PROFILING_START_FORCE_FIRST, "true") @@ -334,6 +336,8 @@ class ConfigTest extends DDSpecification { Config config = new Config() then: + config.apiKey == null // system properties cannot be used to provide a key + config.site == "new site" config.serviceName == "something else" config.traceEnabled == false config.writerType == "LoggingWriter" @@ -372,9 +376,7 @@ class ConfigTest extends DDSpecification { config.traceRateLimit == 200 config.profilingEnabled == true - config.site == "new site" config.profilingUrl == "new url" - config.profilingApiKey == null // system properties cannot be used to provide a key config.mergedProfilingTags == [b: "2", f: "6", (HOST_TAG): "test-host", (RUNTIME_ID_TAG): config.getRuntimeId(), (SERVICE_TAG): config.serviceName, (LANGUAGE_TAG_KEY): LANGUAGE_TAG_VALUE] config.profilingStartDelay == 1111 config.profilingStartForceFirst == true @@ -390,6 +392,7 @@ class ConfigTest extends DDSpecification { def "specify overrides via env vars"() { setup: + environmentVariables.set(DD_API_KEY_ENV, "test-api-key") environmentVariables.set(DD_SERVICE_NAME_ENV, "still something else") environmentVariables.set(DD_TRACE_ENABLED_ENV, "false") environmentVariables.set(DD_WRITER_TYPE_ENV, "LoggingWriter") @@ -397,12 +400,12 @@ class ConfigTest extends DDSpecification { environmentVariables.set(DD_PROPAGATION_STYLE_INJECT, "Datadog B3") environmentVariables.set(DD_JMXFETCH_METRICS_CONFIGS_ENV, "some/file") environmentVariables.set(DD_TRACE_REPORT_HOSTNAME, "true") - environmentVariables.set(DD_PROFILING_API_KEY_ENV, "test-api-key") when: def config = new Config() then: + config.apiKey == "test-api-key" config.serviceName == "still something else" config.traceEnabled == false config.writerType == "LoggingWriter" @@ -410,7 +413,6 @@ class ConfigTest extends DDSpecification { config.propagationStylesToInject.toList() == [Config.PropagationStyle.DATADOG, Config.PropagationStyle.B3] config.jmxFetchMetricsConfigs == ["some/file"] config.reportHostName == true - config.profilingApiKey == "test-api-key" } def "sys props override env vars"() { @@ -1008,14 +1010,14 @@ class ConfigTest extends DDSpecification { def "verify api key loaded from file: #path"() { setup: - environmentVariables.set(DD_PROFILING_API_KEY_ENV, "default-api-key") - System.setProperty(PREFIX + PROFILING_API_KEY_FILE, path) + environmentVariables.set(DD_API_KEY_ENV, "default-api-key") + System.setProperty(PREFIX + API_KEY_FILE, path) when: def config = new Config() then: - config.profilingApiKey == expectedKey + config.apiKey == expectedKey where: path | expectedKey @@ -1023,6 +1025,17 @@ class ConfigTest extends DDSpecification { "/path/that/doesnt/exist" | "default-api-key" } + def "verify api key loaded from old option name"() { + setup: + environmentVariables.set(DD_PROFILING_API_KEY_OLD_ENV, "old-api-key") + + when: + def config = new Config() + + then: + config.apiKey == "old-api-key" + } + def "verify api key loaded from file for old option name: #path"() { setup: environmentVariables.set(DD_PROFILING_API_KEY_OLD_ENV, "default-api-key") @@ -1032,7 +1045,7 @@ class ConfigTest extends DDSpecification { def config = new Config() then: - config.profilingApiKey == expectedKey + config.apiKey == expectedKey where: path | expectedKey @@ -1040,16 +1053,56 @@ class ConfigTest extends DDSpecification { "/path/that/doesnt/exist" | "default-api-key" } - def "verify api key loaded from new option when both new and old are set"() { + def "verify api key loaded from very old option name"() { setup: - System.setProperty(PREFIX + PROFILING_API_KEY_FILE_OLD, getClass().getClassLoader().getResource("apikey.old").getFile()) - System.setProperty(PREFIX + PROFILING_API_KEY_FILE, getClass().getClassLoader().getResource("apikey").getFile()) + environmentVariables.set(DD_PROFILING_API_KEY_VERY_OLD_ENV, "very-old-api-key") when: def config = new Config() then: - config.profilingApiKey == "test-api-key" + config.apiKey == "very-old-api-key" + } + + def "verify api key loaded from file for very old option name: #path"() { + setup: + environmentVariables.set(DD_PROFILING_API_KEY_VERY_OLD_ENV, "default-api-key") + System.setProperty(PREFIX + PROFILING_API_KEY_FILE_VERY_OLD, path) + + when: + def config = new Config() + + then: + config.apiKey == expectedKey + + where: + path | expectedKey + getClass().getClassLoader().getResource("apikey.very-old").getFile() | "test-api-key-very-old" + "/path/that/doesnt/exist" | "default-api-key" + } + + def "verify api key loaded from new option when both new and old are set"() { + setup: + System.setProperty(PREFIX + API_KEY_FILE, getClass().getClassLoader().getResource("apikey").getFile()) + System.setProperty(PREFIX + PROFILING_API_KEY_FILE_OLD, getClass().getClassLoader().getResource("apikey.old").getFile()) + + when: + def config = new Config() + + then: + config.apiKey == "test-api-key" + } + + def "verify api key loaded from new option when both old and very old are set"() { + setup: + System.setProperty(PREFIX + PROFILING_API_KEY_FILE_OLD, getClass().getClassLoader().getResource("apikey.old").getFile()) + System.setProperty(PREFIX + PROFILING_API_KEY_FILE_VERY_OLD, getClass().getClassLoader().getResource("apikey.very-old").getFile()) + + when: + def config = new Config() + + then: + config.apiKey == "test-api-key-old" } def "verify dd.tags overrides global tags in properties"() { @@ -1112,30 +1165,30 @@ class ConfigTest extends DDSpecification { config.mergedProfilingTags == [a: "1", f: "6", (HOST_TAG): config.getHostName(), (RUNTIME_ID_TAG): config.getRuntimeId(), (SERVICE_TAG): config.serviceName, (LANGUAGE_TAG_KEY): LANGUAGE_TAG_VALUE] } - + def "toString works when passwords are empty"() { when: def config = new Config() then: - config.toString().contains("profilingApiKey=null") + config.toString().contains("apiKey=null") config.toString().contains("profilingProxyPassword=null") } def "sensitive information removed for toString/debug log"() { setup: - environmentVariables.set(DD_PROFILING_API_KEY_ENV, "test-secret-api-key") + environmentVariables.set(DD_API_KEY_ENV, "test-secret-api-key") environmentVariables.set(DD_PROFILING_PROXY_PASSWORD_ENV, "test-secret-proxy-password") when: def config = new Config() then: - config.toString().contains("profilingApiKey=****") + config.toString().contains("apiKey=****") !config.toString().contains("test-secret-api-key") config.toString().contains("profilingProxyPassword=****") !config.toString().contains("test-secret-proxy-password") - config.profilingApiKey == "test-secret-api-key" + config.apiKey == "test-secret-api-key" config.profilingProxyPassword == "test-secret-proxy-password" } diff --git a/dd-trace-api/src/test/resources/apikey.very-old b/dd-trace-api/src/test/resources/apikey.very-old new file mode 100644 index 0000000000..ebe7a17f7b --- /dev/null +++ b/dd-trace-api/src/test/resources/apikey.very-old @@ -0,0 +1 @@ +test-api-key-very-old From e7f0d2db122c4d06e46a516a7883997172cad4db Mon Sep 17 00:00:00 2001 From: Tyler Benson Date: Tue, 7 Apr 2020 12:35:20 -0400 Subject: [PATCH 40/43] Alternate fix for #1319 Turns out the issue wasn't with an empty string returning an empty array, but a string with just `,`. ``` jshell> ",".split(",") $1 ==> String[0] { } ``` A non-intuitive edge case in my opinion. --- .../datadog/trace/agent/tooling/OpenTracing32.java | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/dd-java-agent/agent-tooling/src/main/java/datadog/trace/agent/tooling/OpenTracing32.java b/dd-java-agent/agent-tooling/src/main/java/datadog/trace/agent/tooling/OpenTracing32.java index 55b17b5d35..db51e06bcf 100644 --- a/dd-java-agent/agent-tooling/src/main/java/datadog/trace/agent/tooling/OpenTracing32.java +++ b/dd-java-agent/agent-tooling/src/main/java/datadog/trace/agent/tooling/OpenTracing32.java @@ -299,12 +299,15 @@ public final class OpenTracing32 implements TracerAPI { extracted = new HashMap<>(); for (final String key : getter.keys(carrier)) { // extracted header value - String s = getter.get(carrier, key); + String value = getter.get(carrier, key); // in case of multiple values in the header, need to parse - if (s != null && !s.isEmpty()) { - s = s.split(",")[0].trim(); + if (value != null) { + final String[] split = value.split(","); + if (split.length > 0) { + value = split[0].trim(); + } } - extracted.put(key, s); + extracted.put(key, value); } } From a8c14cdcb2b48e2e534263bfc266c2ec5d642cbc Mon Sep 17 00:00:00 2001 From: Kyle Verhoog Date: Tue, 7 Apr 2020 20:58:38 -0400 Subject: [PATCH 41/43] Fix reliability branch filter regex --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 305ebf57ad..74c114cb9d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -360,7 +360,7 @@ workflows: branches: only: - master - - .*-reliability + - /.*-reliability/ - check: requires: From 7f146d74dcf750354081291d9b237fdcce52dcc2 Mon Sep 17 00:00:00 2001 From: Lev Priima <1118651+lpriima@users.noreply.github.com> Date: Fri, 10 Apr 2020 10:50:11 -0700 Subject: [PATCH 42/43] DD_SERVICE; DD_ENV; DD_VERSION env vars support (#1341) DD_SERVICE; DD_ENV; DD_VERSION env vars support * Config.valueOf("", null, "") with throw NPE * mark unused public fields @Deprecated in Config.java * keep unused public fields untouched * retrun 2arg Config ctor --- .../main/java/datadog/trace/api/Config.java | 204 +++++++++++------- .../datadog/trace/api/ConfigTest.groovy | 131 ++++++++++- .../ddagent/TraceProcessingDisruptor.java | 3 +- .../sampling/RateByServiceSamplerTest.groovy | 3 +- 4 files changed, 255 insertions(+), 86 deletions(-) diff --git a/dd-trace-api/src/main/java/datadog/trace/api/Config.java b/dd-trace-api/src/main/java/datadog/trace/api/Config.java index 5a5e0b45cd..5dc753a186 100644 --- a/dd-trace-api/src/main/java/datadog/trace/api/Config.java +++ b/dd-trace-api/src/main/java/datadog/trace/api/Config.java @@ -6,6 +6,8 @@ import java.io.FileNotFoundException; import java.io.FileReader; import java.io.IOException; import java.io.InputStreamReader; +import java.lang.invoke.MethodHandles; +import java.lang.invoke.MethodType; import java.net.InetAddress; import java.net.UnknownHostException; import java.nio.charset.StandardCharsets; @@ -25,6 +27,7 @@ import java.util.SortedSet; import java.util.UUID; import java.util.regex.Pattern; import lombok.Getter; +import lombok.NonNull; import lombok.ToString; import lombok.extern.slf4j.Slf4j; @@ -64,12 +67,13 @@ public class Config { public static final String TRACE_RESOLVER_ENABLED = "trace.resolver.enabled"; public static final String SERVICE_MAPPING = "service.mapping"; + private static final String ENV = "env"; + private static final String VERSION = "version"; public static final String TAGS = "tags"; @Deprecated // Use dd.tags instead public static final String GLOBAL_TAGS = "trace.global.tags"; public static final String SPAN_TAGS = "trace.span.tags"; public static final String JMX_TAGS = "trace.jmx.tags"; - public static final String TRACE_ANALYTICS_ENABLED = "trace.analytics.enabled"; public static final String TRACE_ANNOTATIONS = "trace.annotations"; public static final String TRACE_EXECUTORS_ALL = "trace.executors.all"; @@ -139,7 +143,8 @@ public class Config { public static final String PROFILING_PROXY_PASSWORD = "profiling.proxy.password"; public static final String RUNTIME_ID_TAG = "runtime-id"; - public static final String SERVICE_TAG = "service"; + public static final String SERVICE = "service"; + public static final String SERVICE_TAG = SERVICE; public static final String HOST_TAG = "host"; public static final String LANGUAGE_TAG_KEY = "language"; public static final String LANGUAGE_TAG_VALUE = "jvm"; @@ -338,7 +343,9 @@ public class Config { } } site = getSettingFromEnvironment(SITE, DEFAULT_SITE); - serviceName = getSettingFromEnvironment(SERVICE_NAME, DEFAULT_SERVICE_NAME); + serviceName = + getSettingFromEnvironment( + SERVICE_NAME, getSettingFromEnvironment(SERVICE, DEFAULT_SERVICE_NAME)); traceEnabled = getBooleanSettingFromEnvironment(TRACE_ENABLED, DEFAULT_TRACE_ENABLED); integrationsEnabled = @@ -357,7 +364,10 @@ public class Config { getBooleanSettingFromEnvironment(TRACE_RESOLVER_ENABLED, DEFAULT_TRACE_RESOLVER_ENABLED); serviceMapping = getMapSettingFromEnvironment(SERVICE_MAPPING, null); - tags = getMapSettingFromEnvironment(TAGS, null); + final Map tagsPreMap = new HashMap<>(getMapSettingFromEnvironment(TAGS, null)); + addPropToMapIfDefinedByEnvironment(tagsPreMap, ENV); + addPropToMapIfDefinedByEnvironment(tagsPreMap, VERSION); + tags = Collections.unmodifiableMap(tagsPreMap); globalTags = getMapSettingFromEnvironment(GLOBAL_TAGS, null); spanTags = getMapSettingFromEnvironment(SPAN_TAGS, null); jmxTags = getMapSettingFromEnvironment(JMX_TAGS, null); @@ -405,17 +415,11 @@ public class Config { RUNTIME_CONTEXT_FIELD_INJECTION, DEFAULT_RUNTIME_CONTEXT_FIELD_INJECTION); propagationStylesToExtract = - getEnumSetSettingFromEnvironment( - PROPAGATION_STYLE_EXTRACT, - DEFAULT_PROPAGATION_STYLE_EXTRACT, - PropagationStyle.class, - true); + getPropagationStyleSetSettingFromEnvironmentOrDefault( + PROPAGATION_STYLE_EXTRACT, DEFAULT_PROPAGATION_STYLE_EXTRACT); propagationStylesToInject = - getEnumSetSettingFromEnvironment( - PROPAGATION_STYLE_INJECT, - DEFAULT_PROPAGATION_STYLE_INJECT, - PropagationStyle.class, - true); + getPropagationStyleSetSettingFromEnvironmentOrDefault( + PROPAGATION_STYLE_INJECT, DEFAULT_PROPAGATION_STYLE_INJECT); jmxFetchEnabled = getBooleanSettingFromEnvironment(JMX_FETCH_ENABLED, DEFAULT_JMX_FETCH_ENABLED); @@ -597,13 +601,13 @@ public class Config { properties, RUNTIME_CONTEXT_FIELD_INJECTION, parent.runtimeContextFieldInjection); final Set parsedPropagationStylesToExtract = - getPropertySetValue(properties, PROPAGATION_STYLE_EXTRACT, PropagationStyle.class); + getPropagationStyleSetFromPropertyValue(properties, PROPAGATION_STYLE_EXTRACT); propagationStylesToExtract = parsedPropagationStylesToExtract == null ? parent.propagationStylesToExtract : parsedPropagationStylesToExtract; final Set parsedPropagationStylesToInject = - getPropertySetValue(properties, PROPAGATION_STYLE_INJECT, PropagationStyle.class); + getPropagationStyleSetFromPropertyValue(properties, PROPAGATION_STYLE_INJECT); propagationStylesToInject = parsedPropagationStylesToInject == null ? parent.propagationStylesToInject @@ -803,7 +807,8 @@ public class Config { * @deprecated This method should only be used internally. Use the instance getter instead {@link * #isIntegrationEnabled(SortedSet, boolean)}. */ - public static boolean integrationEnabled( + @Deprecated + private static boolean integrationEnabled( final SortedSet integrationNames, final boolean defaultEnabled) { // If default is enabled, we want to enable individually, // if default is disabled, we want to disable individually. @@ -896,9 +901,10 @@ public class Config { */ private static String getSettingFromEnvironment(final String name, final String defaultValue) { String value; + final String systemPropertyName = propertyNameToSystemPropertyName(name); // System properties and properties provided from command line have the highest precedence - value = System.getProperties().getProperty(propertyNameToSystemPropertyName(name)); + value = System.getProperties().getProperty(systemPropertyName); if (null != value) { return value; } @@ -910,7 +916,7 @@ public class Config { } // If value is not defined yet, we look at properties optionally defined in a properties file - value = propertiesFromConfigFile.getProperty(propertyNameToSystemPropertyName(name)); + value = propertiesFromConfigFile.getProperty(systemPropertyName); if (null != value) { return value; } @@ -919,6 +925,7 @@ public class Config { } /** @deprecated This method should only be used internally. Use the explicit getter instead. */ + @NonNull private static Map getMapSettingFromEnvironment( final String name, final String defaultValue) { return parseMap( @@ -931,7 +938,8 @@ public class Config { * * @deprecated This method should only be used internally. Use the explicit getter instead. */ - public static List getListSettingFromEnvironment( + @NonNull + private static List getListSettingFromEnvironment( final String name, final String defaultValue) { return parseList(getSettingFromEnvironment(name, defaultValue)); } @@ -943,8 +951,7 @@ public class Config { */ public static Boolean getBooleanSettingFromEnvironment( final String name, final Boolean defaultValue) { - final String value = getSettingFromEnvironment(name, null); - return value == null || value.trim().isEmpty() ? defaultValue : Boolean.valueOf(value); + return getSettingFromEnvironmentWithLog(name, Boolean.class, defaultValue); } /** @@ -953,13 +960,7 @@ public class Config { * @deprecated This method should only be used internally. Use the explicit getter instead. */ public static Float getFloatSettingFromEnvironment(final String name, final Float defaultValue) { - final String value = getSettingFromEnvironment(name, null); - try { - return value == null ? defaultValue : Float.valueOf(value); - } catch (final NumberFormatException e) { - log.warn("Invalid configuration for " + name, e); - return defaultValue; - } + return getSettingFromEnvironmentWithLog(name, Float.class, defaultValue); } /** @@ -967,15 +968,10 @@ public class Config { * * @deprecated This method should only be used internally. Use the explicit getter instead. */ - public static Double getDoubleSettingFromEnvironment( + @Deprecated + private static Double getDoubleSettingFromEnvironment( final String name, final Double defaultValue) { - final String value = getSettingFromEnvironment(name, null); - try { - return value == null ? defaultValue : Double.valueOf(value); - } catch (final NumberFormatException e) { - log.warn("Invalid configuration for " + name, e); - return defaultValue; - } + return getSettingFromEnvironmentWithLog(name, Double.class, defaultValue); } /** @@ -983,9 +979,13 @@ public class Config { */ private static Integer getIntegerSettingFromEnvironment( final String name, final Integer defaultValue) { - final String value = getSettingFromEnvironment(name, null); + return getSettingFromEnvironmentWithLog(name, Integer.class, defaultValue); + } + + private static T getSettingFromEnvironmentWithLog( + final String name, Class tClass, final T defaultValue) { try { - return value == null ? defaultValue : Integer.valueOf(value); + return valueOf(getSettingFromEnvironment(name, null), tClass, defaultValue); } catch (final NumberFormatException e) { log.warn("Invalid configuration for " + name, e); return defaultValue; @@ -996,28 +996,22 @@ public class Config { * Calls {@link #getSettingFromEnvironment(String, String)} and converts the result to a set of * strings splitting by space or comma. */ - private static > Set getEnumSetSettingFromEnvironment( - final String name, - final String defaultValue, - final Class clazz, - final boolean emptyResultMeansUseDefault) { + private static Set getPropagationStyleSetSettingFromEnvironmentOrDefault( + final String name, final String defaultValue) { final String value = getSettingFromEnvironment(name, defaultValue); - Set result = - convertStringSetToEnumSet( - parseStringIntoSetOfNonEmptyStrings(value, SPLIT_BY_SPACE_OR_COMMA_REGEX), clazz); + Set result = + convertStringSetToPropagationStyleSet(parseStringIntoSetOfNonEmptyStrings(value)); - if (emptyResultMeansUseDefault && result.isEmpty()) { + if (result.isEmpty()) { // Treat empty parsing result as no value and use default instead result = - convertStringSetToEnumSet( - parseStringIntoSetOfNonEmptyStrings(defaultValue, SPLIT_BY_SPACE_OR_COMMA_REGEX), - clazz); + convertStringSetToPropagationStyleSet(parseStringIntoSetOfNonEmptyStrings(defaultValue)); } return result; } - private Set getIntegerRangeSettingFromEnvironment( + private static Set getIntegerRangeSettingFromEnvironment( final String name, final Set defaultValue) { final String value = getSettingFromEnvironment(name, null); try { @@ -1035,6 +1029,7 @@ public class Config { * @param setting The setting name, e.g. `service.name` * @return The public facing environment variable name */ + @NonNull private static String propertyNameToEnvironmentVariableName(final String setting) { return ENV_REPLACEMENT .matcher(propertyNameToSystemPropertyName(setting).toUpperCase()) @@ -1048,10 +1043,41 @@ public class Config { * @param setting The setting name, e.g. `service.name` * @return The public facing system property name */ + @NonNull private static String propertyNameToSystemPropertyName(final String setting) { return PREFIX + setting; } + /** + * @param value to parse by tClass::valueOf + * @param tClass should contain static parsing method "T valueOf(String)" + * @param defaultValue + * @param + * @return value == null || value.trim().isEmpty() ? defaultValue : tClass.valueOf(value) + * @throws NumberFormatException + */ + private static T valueOf( + final String value, @NonNull final Class tClass, final T defaultValue) { + if (value == null || value.trim().isEmpty()) { + log.debug("valueOf: using defaultValue '{}' for '{}' of '{}' ", defaultValue, value, tClass); + return defaultValue; + } + try { + return (T) + MethodHandles.publicLookup() + .findStatic(tClass, "valueOf", MethodType.methodType(tClass, String.class)) + .invoke(value); + } catch (NumberFormatException e) { + throw e; + } catch (NoSuchMethodException | IllegalAccessException e) { + log.debug("Can't invoke or access 'valueOf': ", e); + throw new NumberFormatException(e.toString()); + } catch (Throwable e) { + log.debug("Can't parse: ", e); + throw new NumberFormatException(e.toString()); + } + } + private static Map getPropertyMapValue( final Properties properties, final String name, final Map defaultValue) { final String value = properties.getProperty(name); @@ -1066,29 +1092,25 @@ public class Config { private static Boolean getPropertyBooleanValue( final Properties properties, final String name, final Boolean defaultValue) { - final String value = properties.getProperty(name); - return value == null || value.trim().isEmpty() ? defaultValue : Boolean.valueOf(value); + return valueOf(properties.getProperty(name), Boolean.class, defaultValue); } private static Integer getPropertyIntegerValue( final Properties properties, final String name, final Integer defaultValue) { - final String value = properties.getProperty(name); - return value == null || value.trim().isEmpty() ? defaultValue : Integer.valueOf(value); + return valueOf(properties.getProperty(name), Integer.class, defaultValue); } private static Double getPropertyDoubleValue( final Properties properties, final String name, final Double defaultValue) { - final String value = properties.getProperty(name); - return value == null || value.trim().isEmpty() ? defaultValue : Double.valueOf(value); + return valueOf(properties.getProperty(name), Double.class, defaultValue); } - private static > Set getPropertySetValue( - final Properties properties, final String name, final Class clazz) { + private static Set getPropagationStyleSetFromPropertyValue( + final Properties properties, final String name) { final String value = properties.getProperty(name); if (value != null) { - final Set result = - convertStringSetToEnumSet( - parseStringIntoSetOfNonEmptyStrings(value, SPLIT_BY_SPACE_OR_COMMA_REGEX), clazz); + final Set result = + convertStringSetToPropagationStyleSet(parseStringIntoSetOfNonEmptyStrings(value)); if (!result.isEmpty()) { return result; } @@ -1097,7 +1119,7 @@ public class Config { return null; } - private Set getPropertyIntegerRangeValue( + private static Set getPropertyIntegerRangeValue( final Properties properties, final String name, final Set defaultValue) { final String value = properties.getProperty(name); try { @@ -1108,6 +1130,7 @@ public class Config { } } + @NonNull private static Map parseMap(final String str, final String settingName) { // If we ever want to have default values besides an empty map, this will need to change. if (str == null || str.trim().isEmpty()) { @@ -1137,9 +1160,9 @@ public class Config { return Collections.unmodifiableMap(map); } - private static Set parseIntegerRangeSet(String str, final String settingName) + @NonNull + private static Set parseIntegerRangeSet(@NonNull String str, final String settingName) throws NumberFormatException { - assert str != null; str = str.replaceAll("\\s", ""); if (!str.matches("\\d{3}(?:-\\d{3})?(?:,\\d{3}(?:-\\d{3})?)*")) { log.warn( @@ -1169,10 +1192,26 @@ public class Config { return Collections.unmodifiableSet(set); } + @NonNull private static Map newHashMap(final int size) { return new HashMap<>(size + 1, 1f); } + /** + * @param map + * @param propName + * @return true if map was modified + */ + private static boolean addPropToMapIfDefinedByEnvironment( + final Map map, final String propName) { + final String val = getSettingFromEnvironment(propName, null); + if (val != null) { + return !val.equals(map.put(propertyNameToSystemPropertyName(propName), val)); + } + return false; + } + + @NonNull private static List parseList(final String str) { if (str == null || str.trim().isEmpty()) { return Collections.emptyList(); @@ -1186,13 +1225,13 @@ public class Config { return Collections.unmodifiableList(Arrays.asList(tokens)); } - private static Set parseStringIntoSetOfNonEmptyStrings( - final String str, final String regex) { + @NonNull + private static Set parseStringIntoSetOfNonEmptyStrings(final String str) { // Using LinkedHashSet to preserve original string order final Set result = new LinkedHashSet<>(); // Java returns single value when splitting an empty string. We do not need that value, so // we need to throw it out. - for (final String value : str.split(regex)) { + for (final String value : str.split(SPLIT_BY_SPACE_OR_COMMA_REGEX)) { if (!value.isEmpty()) { result.add(value); } @@ -1200,15 +1239,16 @@ public class Config { return Collections.unmodifiableSet(result); } - private static > Set convertStringSetToEnumSet( - final Set input, final Class clazz) { + @NonNull + private static Set convertStringSetToPropagationStyleSet( + final Set input) { // Using LinkedHashSet to preserve original string order - final Set result = new LinkedHashSet<>(); + final Set result = new LinkedHashSet<>(); for (final String value : input) { try { - result.add(Enum.valueOf(clazz, value.toUpperCase())); + result.add(PropagationStyle.valueOf(value.toUpperCase())); } catch (final IllegalArgumentException e) { - log.debug("Cannot recognize config string value: {}, {}", value, clazz); + log.debug("Cannot recognize config string value: {}, {}", value, PropagationStyle.class); } } return Collections.unmodifiableSet(result); @@ -1245,8 +1285,7 @@ public class Config { return properties; } - try { - final FileReader fileReader = new FileReader(configurationFile); + try (final FileReader fileReader = new FileReader(configurationFile)) { properties.load(fileReader); } catch (final FileNotFoundException fnf) { log.error("Configuration file '{}' not found.", configurationFilePath); @@ -1259,8 +1298,8 @@ public class Config { } /** Returns the detected hostname. First tries locally, then using DNS */ - private String getHostName() { - String possibleHostname = null; + private static String getHostName() { + String possibleHostname; // Try environment variable. This works in almost all environments if (System.getProperty("os.name").startsWith("Windows")) { @@ -1275,12 +1314,11 @@ public class Config { } // Try hostname command - try { - final Process process = Runtime.getRuntime().exec("hostname"); - final BufferedReader reader = - new BufferedReader(new InputStreamReader(process.getInputStream())); + try (final BufferedReader reader = + new BufferedReader( + new InputStreamReader(Runtime.getRuntime().exec("hostname").getInputStream()))) { possibleHostname = reader.readLine(); - } catch (final Exception e) { + } catch (final Exception ignore) { // Ignore. Hostname command is not always available } diff --git a/dd-trace-api/src/test/groovy/datadog/trace/api/ConfigTest.groovy b/dd-trace-api/src/test/groovy/datadog/trace/api/ConfigTest.groovy index 5b1bdf15fd..267abf6381 100644 --- a/dd-trace-api/src/test/groovy/datadog/trace/api/ConfigTest.groovy +++ b/dd-trace-api/src/test/groovy/datadog/trace/api/ConfigTest.groovy @@ -82,6 +82,8 @@ class ConfigTest extends DDSpecification { private static final DD_WRITER_TYPE_ENV = "DD_WRITER_TYPE" private static final DD_SERVICE_MAPPING_ENV = "DD_SERVICE_MAPPING" private static final DD_TAGS_ENV = "DD_TAGS" + private static final DD_ENV_ENV = "DD_ENV" + private static final DD_VERSION_ENV = "DD_VERSION" private static final DD_GLOBAL_TAGS_ENV = "DD_TRACE_GLOBAL_TAGS" private static final DD_SPAN_TAGS_ENV = "DD_TRACE_SPAN_TAGS" private static final DD_HEADER_TAGS_ENV = "DD_TRACE_HEADER_TAGS" @@ -951,6 +953,7 @@ class ConfigTest extends DDSpecification { setup: System.setProperty(PREFIX + CONFIGURATION_FILE, "src/test/resources/dd-java-tracer.properties") environmentVariables.set("DD_SERVICE_NAME", "set-in-env") + environmentVariables.set("DD_SERVICE", "some-other-ignored-name") when: def config = new Config() @@ -961,7 +964,17 @@ class ConfigTest extends DDSpecification { cleanup: System.clearProperty(PREFIX + CONFIGURATION_FILE) System.clearProperty(PREFIX + SERVICE_NAME) - environmentVariables.clear("DD_SERVICE_NAME") + } + + def "verify fallback to DD_SERVICE"() { + setup: + environmentVariables.set("DD_SERVICE", "service-name-from-dd-service-env-var") + + when: + def config = new Config() + + then: + config.serviceName == "service-name-from-dd-service-env-var" } def "verify fallback to properties file that does not exist does not crash app"() { @@ -1218,4 +1231,120 @@ class ConfigTest extends DDSpecification { config.getFinalProfilingUrl() == "https://some.new.url/goes/here" } + def "fallback to DD_TAGS"() { + setup: + environmentVariables.set(DD_TAGS_ENV, "a:1,b:2,c:3") + + when: + Config config = new Config() + + then: + config.mergedSpanTags == [a: "1", c: "3", b: "2"] + } + + def "precedence of DD_ENV and DD_VERSION"() { + setup: + environmentVariables.set(DD_ENV_ENV, "test_env") + environmentVariables.set(DD_VERSION_ENV, "1.2.3") + environmentVariables.set(DD_TAGS_ENV, "dd.env:production , dd.version:3.2.1") + + when: + Config config = new Config() + + then: + config.mergedSpanTags == ["dd.env": "test_env", "dd.version": "1.2.3"] + } + + def "propertyNameToEnvironmentVariableName unit test"() { + expect: + Config.propertyNameToEnvironmentVariableName(Config.SERVICE) == "DD_SERVICE" + } + + def "getProperty*Value unit test"() { + setup: + def p = new Properties() + p.setProperty("a", "42.42") + p.setProperty("intProp", "13") + + expect: + Config.getPropertyDoubleValue(p, "intProp", 40) == 13 + Config.getPropertyDoubleValue(p, "a", 41) == 42.42 + Config.getPropertyIntegerValue(p, "b", 61) == 61 + Config.getPropertyIntegerValue(p, "intProp", 61) == 13 + Config.getPropertyBooleanValue(p, "a", true) == false + } + + def "valueOf positive test"() { + expect: + Config.valueOf(value, tClass, defaultValue) == expected + + where: + value | tClass | defaultValue | expected + "42.42" | Boolean | true | false + "42.42" | Boolean | null | false + "true" | Boolean | null | true + "trUe" | Boolean | null | true + "trUe" | Boolean | false | true + "tru" | Boolean | true | false + "truee" | Boolean | true | false + "true " | Boolean | true | false + " true" | Boolean | true | false + " true " | Boolean | true | false + " true "| Boolean | true | false + null | Float | 43.3 | 43.3 + "42.42" | Float | 21.21 | 42.42f + null | Double | 43.3 | 43.3 + "42.42" | Double | 21.21 | 42.42 + null | Integer | 13 | 13 + "44" | Integer | 21 | 44 + "45" | Long | 21 | 45 + "46" | Short | 21 | 46 + } + + def "valueOf negative test when tClass is null"() { + when: + Config.valueOf(value, tClass, defaultValue) + + then: + def exception = thrown(NullPointerException) + exception.message == "tClass is marked non-null but is null" + + where: + value | tClass | defaultValue + null | null | "42" + "" | null | "43" + " " | null | "44" + "1" | null | "45" + } + + def "valueOf negative test"() { + when: + Config.valueOf(value, tClass, null) + + then: + def exception = thrown(NumberFormatException) + println("cause: " : exception.message) + + where: + value | tClass + "42.42" | Number + "42.42" | Byte + "42.42" | Character + "42.42" | Short + "42.42" | Integer + "42.42" | Long + "42.42" | Object + "42.42" | Object[] + "42.42" | boolean[] + "42.42" | boolean + "42.42" | byte + "42.42" | byte + "42.42" | char + "42.42" | short + "42.42" | int + "42.42" | long + "42.42" | double + "42.42" | float + } + } diff --git a/dd-trace-ot/src/main/java/datadog/trace/common/writer/ddagent/TraceProcessingDisruptor.java b/dd-trace-ot/src/main/java/datadog/trace/common/writer/ddagent/TraceProcessingDisruptor.java index 86d0e7177b..67c8b91adf 100644 --- a/dd-trace-ot/src/main/java/datadog/trace/common/writer/ddagent/TraceProcessingDisruptor.java +++ b/dd-trace-ot/src/main/java/datadog/trace/common/writer/ddagent/TraceProcessingDisruptor.java @@ -3,6 +3,7 @@ package datadog.trace.common.writer.ddagent; import com.lmax.disruptor.EventHandler; import datadog.common.exec.DaemonThreadFactory; import datadog.opentracing.DDSpan; +import datadog.opentracing.DDSpanContext; import datadog.trace.common.writer.DDAgentWriter; import java.util.List; import lombok.extern.slf4j.Slf4j; @@ -65,7 +66,7 @@ public class TraceProcessingDisruptor extends AbstractDisruptor> { // attempt to have agent scale the metrics properly ((DDSpan) event.data.get(0).getLocalRootSpan()) .context() - .setMetric("_sample_rate", 1d / event.representativeCount); + .setMetric(DDSpanContext.SAMPLE_RATE_KEY, 1d / event.representativeCount); } try { final byte[] serializedTrace = api.serializeTrace(event.data); diff --git a/dd-trace-ot/src/test/groovy/datadog/trace/api/sampling/RateByServiceSamplerTest.groovy b/dd-trace-ot/src/test/groovy/datadog/trace/api/sampling/RateByServiceSamplerTest.groovy index 5b147764f3..4ecf463700 100644 --- a/dd-trace-ot/src/test/groovy/datadog/trace/api/sampling/RateByServiceSamplerTest.groovy +++ b/dd-trace-ot/src/test/groovy/datadog/trace/api/sampling/RateByServiceSamplerTest.groovy @@ -2,6 +2,7 @@ package datadog.trace.api.sampling import datadog.opentracing.DDSpan +import datadog.opentracing.DDSpanContext import datadog.opentracing.DDTracer import datadog.opentracing.SpanFactory import datadog.trace.api.DDTags @@ -68,7 +69,7 @@ class RateByServiceSamplerTest extends DDSpecification { // sets correctly on root span span.getSamplingPriority() == PrioritySampling.SAMPLER_KEEP // RateByServiceSamler must not set the sample rate - span.getMetrics().get("_sample_rate") == null + span.getMetrics().get(DDSpanContext.SAMPLE_RATE_KEY) == null } def "sampling priority set when service later"() { From b85e023fbec6f104e7533b6b91a9af75742bb84e Mon Sep 17 00:00:00 2001 From: Tyler Benson Date: Fri, 10 Apr 2020 13:56:03 -0400 Subject: [PATCH 43/43] Revert "add ..."metrics":{"_dd.measured":1}... to every span (#1336)" This reverts commit cda39adf50bc5ec9b74117292dc0e67332b4777a. --- .../src/main/java/datadog/trace/api/Config.java | 2 +- .../main/java/datadog/opentracing/DDSpanContext.java | 9 +++------ .../opentracing/DDSpanSerializationTest.groovy | 1 - .../opentracing/decorators/SpanDecoratorTest.groovy | 5 +---- .../groovy/datadog/trace/DDSpanContextTest.groovy | 11 +++-------- .../datadog/trace/api/writer/DDAgentApiTest.groovy | 6 ++---- .../datadog/trace/api/writer/LoggingWriterTest.groovy | 2 +- 7 files changed, 11 insertions(+), 25 deletions(-) diff --git a/dd-trace-api/src/main/java/datadog/trace/api/Config.java b/dd-trace-api/src/main/java/datadog/trace/api/Config.java index 5dc753a186..4320b09e0d 100644 --- a/dd-trace-api/src/main/java/datadog/trace/api/Config.java +++ b/dd-trace-api/src/main/java/datadog/trace/api/Config.java @@ -899,7 +899,7 @@ public class Config { * @return * @deprecated This method should only be used internally. Use the explicit getter instead. */ - private static String getSettingFromEnvironment(final String name, final String defaultValue) { + public static String getSettingFromEnvironment(final String name, final String defaultValue) { String value; final String systemPropertyName = propertyNameToSystemPropertyName(name); diff --git a/dd-trace-ot/src/main/java/datadog/opentracing/DDSpanContext.java b/dd-trace-ot/src/main/java/datadog/opentracing/DDSpanContext.java index 38ad5535d4..88e0974135 100644 --- a/dd-trace-ot/src/main/java/datadog/opentracing/DDSpanContext.java +++ b/dd-trace-ot/src/main/java/datadog/opentracing/DDSpanContext.java @@ -25,11 +25,8 @@ public class DDSpanContext implements io.opentracing.SpanContext { public static final String PRIORITY_SAMPLING_KEY = "_sampling_priority_v1"; public static final String SAMPLE_RATE_KEY = "_sample_rate"; public static final String ORIGIN_KEY = "_dd.origin"; - public static final String DD_MEASURED = "_dd.measured"; - public static final Number DD_MEASURED_DEFAULT = 1; - private static final Map DEFAULT_METRICS = - Collections.singletonMap(DD_MEASURED, DD_MEASURED_DEFAULT); + private static final Map EMPTY_METRICS = Collections.emptyMap(); // Shared with other span contexts /** For technical reasons, the ref to the original tracer */ @@ -310,12 +307,12 @@ public class DDSpanContext implements io.opentracing.SpanContext { public Map getMetrics() { final Map metrics = this.metrics.get(); - return metrics == null ? DEFAULT_METRICS : metrics; + return metrics == null ? EMPTY_METRICS : metrics; } public void setMetric(final String key, final Number value) { if (metrics.get() == null) { - metrics.compareAndSet(null, new ConcurrentHashMap<>(DEFAULT_METRICS)); + metrics.compareAndSet(null, new ConcurrentHashMap()); } if (value instanceof Float) { metrics.get().put(key, value.doubleValue()); diff --git a/dd-trace-ot/src/test/groovy/datadog/opentracing/DDSpanSerializationTest.groovy b/dd-trace-ot/src/test/groovy/datadog/opentracing/DDSpanSerializationTest.groovy index 89f1c00742..fa8d46580a 100644 --- a/dd-trace-ot/src/test/groovy/datadog/opentracing/DDSpanSerializationTest.groovy +++ b/dd-trace-ot/src/test/groovy/datadog/opentracing/DDSpanSerializationTest.groovy @@ -20,7 +20,6 @@ class DDSpanSerializationTest extends DDSpecification { def jsonAdapter = new Moshi.Builder().build().adapter(Map) final Map metrics = ["_sampling_priority_v1": 1] - metrics.putAll(DDSpanContext.DEFAULT_METRICS) if (samplingPriority == PrioritySampling.UNSET) { // RateByServiceSampler sets priority metrics.put("_dd.agent_psr", 1.0d) } diff --git a/dd-trace-ot/src/test/groovy/datadog/opentracing/decorators/SpanDecoratorTest.groovy b/dd-trace-ot/src/test/groovy/datadog/opentracing/decorators/SpanDecoratorTest.groovy index 8725967f40..094b5d77ce 100644 --- a/dd-trace-ot/src/test/groovy/datadog/opentracing/decorators/SpanDecoratorTest.groovy +++ b/dd-trace-ot/src/test/groovy/datadog/opentracing/decorators/SpanDecoratorTest.groovy @@ -279,13 +279,10 @@ class SpanDecoratorTest extends DDSpecification { def "span metrics starts empty but added with rate limiting value of #rate"() { expect: - span.metrics == DDSpanContext.DEFAULT_METRICS + span.metrics == [:] when: span.setTag(ANALYTICS_SAMPLE_RATE, rate) - // these 2 lines to avoid checking for {@code DDSpanContext.DD_MEASURED} for every metric: - span.context().setMetric(DDSpanContext.DD_MEASURED, 42) - span.metrics.remove(DDSpanContext.DD_MEASURED, 42) then: span.metrics == result diff --git a/dd-trace-ot/src/test/groovy/datadog/trace/DDSpanContextTest.groovy b/dd-trace-ot/src/test/groovy/datadog/trace/DDSpanContextTest.groovy index 55ac7a945b..ce2b423829 100644 --- a/dd-trace-ot/src/test/groovy/datadog/trace/DDSpanContextTest.groovy +++ b/dd-trace-ot/src/test/groovy/datadog/trace/DDSpanContextTest.groovy @@ -1,6 +1,5 @@ package datadog.trace -import datadog.opentracing.DDSpanContext import datadog.opentracing.SpanFactory import datadog.trace.api.DDTags import datadog.trace.util.test.DDSpecification @@ -19,7 +18,7 @@ class DDSpanContextTest extends DDSpecification { context.serviceName == "fakeService" context.resourceName == "fakeResource" context.spanType == "fakeType" - context.toString() == "DDSpan [ t_id=1, s_id=1, p_id=0] trace=fakeService/fakeOperation/fakeResource metrics=${defaultMetrics()} *errored* tags={${extra}${tags.containsKey(DDTags.SPAN_TYPE) ? "span.type=${context.getSpanType()}, " : ""}thread.id=${Thread.currentThread().id}, thread.name=${Thread.currentThread().name}}" + context.toString() == "DDSpan [ t_id=1, s_id=1, p_id=0] trace=fakeService/fakeOperation/fakeResource metrics={} *errored* tags={${extra}${tags.containsKey(DDTags.SPAN_TYPE) ? "span.type=${context.getSpanType()}, " : ""}thread.id=${Thread.currentThread().id}, thread.name=${Thread.currentThread().name}}" where: name | extra | tags @@ -36,7 +35,7 @@ class DDSpanContextTest extends DDSpecification { def thread = Thread.currentThread() def expectedTags = [(DDTags.THREAD_NAME): thread.name, (DDTags.THREAD_ID): thread.id] - def expectedTrace = "DDSpan [ t_id=1, s_id=1, p_id=0] trace=$details metrics=${defaultMetrics()} tags={thread.id=$thread.id, thread.name=$thread.name}" + def expectedTrace = "DDSpan [ t_id=1, s_id=1, p_id=0] trace=$details metrics={} tags={thread.id=$thread.id, thread.name=$thread.name}" expect: context.getTags() == expectedTags @@ -62,7 +61,7 @@ class DDSpanContextTest extends DDSpecification { (DDTags.THREAD_NAME): thread.name, (DDTags.THREAD_ID) : thread.id ] - context.toString() == "DDSpan [ t_id=1, s_id=1, p_id=0] trace=fakeService/fakeOperation/fakeResource metrics=${defaultMetrics()} tags={$name=$value, thread.id=$thread.id, thread.name=$thread.name}" + context.toString() == "DDSpan [ t_id=1, s_id=1, p_id=0] trace=fakeService/fakeOperation/fakeResource metrics={} tags={$name=$value, thread.id=$thread.id, thread.name=$thread.name}" where: name | value @@ -99,8 +98,4 @@ class DDSpanContextTest extends DDSpecification { Double | 0.5d Integer | 0x55 } - - static String defaultMetrics() { - return DDSpanContext.DEFAULT_METRICS - } } diff --git a/dd-trace-ot/src/test/groovy/datadog/trace/api/writer/DDAgentApiTest.groovy b/dd-trace-ot/src/test/groovy/datadog/trace/api/writer/DDAgentApiTest.groovy index 6cff04562d..189dca2664 100644 --- a/dd-trace-ot/src/test/groovy/datadog/trace/api/writer/DDAgentApiTest.groovy +++ b/dd-trace-ot/src/test/groovy/datadog/trace/api/writer/DDAgentApiTest.groovy @@ -2,7 +2,6 @@ package datadog.trace.api.writer import com.fasterxml.jackson.core.type.TypeReference import com.fasterxml.jackson.databind.ObjectMapper -import datadog.opentracing.DDSpanContext import datadog.opentracing.SpanFactory import datadog.trace.common.writer.ddagent.DDAgentApi import datadog.trace.common.writer.ddagent.DDAgentResponseListener @@ -102,8 +101,7 @@ class DDAgentApiTest extends DDSpecification { "duration" : 0, "error" : 0, "meta" : ["thread.name": Thread.currentThread().getName(), "thread.id": "${Thread.currentThread().id}"], - //TODO : use DDSpanContext.DD_MEASURED as a key - "metrics" : ["_dd.measured": DDSpanContext.DD_MEASURED_DEFAULT], + "metrics" : [:], "name" : "fakeOperation", "parent_id": 0, "resource" : "fakeResource", @@ -117,7 +115,7 @@ class DDAgentApiTest extends DDSpecification { "duration" : 0, "error" : 0, "meta" : ["thread.name": Thread.currentThread().getName(), "thread.id": "${Thread.currentThread().id}"], - "metrics" : ["_dd.measured": DDSpanContext.DD_MEASURED_DEFAULT], + "metrics" : [:], "name" : "fakeOperation", "parent_id": 0, "resource" : "my-resource", diff --git a/dd-trace-ot/src/test/groovy/datadog/trace/api/writer/LoggingWriterTest.groovy b/dd-trace-ot/src/test/groovy/datadog/trace/api/writer/LoggingWriterTest.groovy index b981784a26..63cacf0684 100644 --- a/dd-trace-ot/src/test/groovy/datadog/trace/api/writer/LoggingWriterTest.groovy +++ b/dd-trace-ot/src/test/groovy/datadog/trace/api/writer/LoggingWriterTest.groovy @@ -16,6 +16,6 @@ class LoggingWriterTest extends DDSpecification { def "test toString"() { expect: - writer.toString(sampleTrace).startsWith('[{"service":"fakeService","name":"fakeOperation","resource":"fakeResource","trace_id":1,"span_id":1,"parent_id":0,"start":1000,"duration":0,"type":"fakeType","error":0,"metrics":{"_dd.measured":1},"meta":{') + writer.toString(sampleTrace).startsWith('[{"service":"fakeService","name":"fakeOperation","resource":"fakeResource","trace_id":1,"span_id":1,"parent_id":0,"start":1000,"duration":0,"type":"fakeType","error":0,"metrics":{},"meta":{') } }