Add config for tags that propagate to JMXFetch and spans

Also add runtime-id tag to root span and JMXFetch metrics

dd-java-agent/agent-jmxfetch/agent-jmxfetch.gradle

@@ -5,7 +5,7 @@ apply from: "${rootDir}/gradle/java.gradle"
 
 dependencies {
   compile 'com.heliosapm.jmxlocal:jmxlocal:1.0'
-  compile 'com.datadoghq:jmxfetch:0.21.0'
+  compile 'com.datadoghq:jmxfetch:0.22.0'
   compile deps.slf4j
   compile project(':dd-trace-api')
 }

dd-java-agent/agent-jmxfetch/src/main/java/datadog/trace/agent/jmxfetch/JMXFetch.java

@@ -3,6 +3,7 @@ package datadog.trace.agent.jmxfetch;
 import com.google.common.collect.ImmutableList;
 import datadog.trace.api.Config;
 import java.util.List;
+import java.util.Map;
 import lombok.extern.slf4j.Slf4j;
 import org.datadog.jmxfetch.App;
 import org.datadog.jmxfetch.AppConfig;
@@ -29,15 +30,17 @@ public class JMXFetch {
     final List<String> metricsConfigs = config.getJmxFetchMetricsConfigs();
     final Integer checkPeriod = config.getJmxFetchCheckPeriod();
     final Integer refreshBeansPeriod = config.getJmxFetchRefreshBeansPeriod();
+    final Map<String, String> globalTags = config.getMergedJmxTags();
     final String reporter = getReporter(config);
     final String logLocation = getLogLocation();
     final String logLevel = getLogLevel();
 
     log.error(
-        "JMXFetch config: {} {} {} {} {} {}",
+        "JMXFetch config: {} {} {} {} {} {} {}",
         metricsConfigs,
         checkPeriod,
         refreshBeansPeriod,
+        globalTags,
         reporter,
         logLocation,
         logLevel);
@@ -47,6 +50,7 @@ public class JMXFetch {
             metricsConfigs,
             checkPeriod,
             refreshBeansPeriod,
+            globalTags,
             reporter,
             logLocation,
             logLevel);

dd-java-agent/instrumentation/akka-http-10.0/src/test/groovy/AkkaHttpServerInstrumentationTest.groovy

@@ -52,7 +52,7 @@ class AkkaHttpServerInstrumentationTest extends AgentTestRunner {
           spanType DDSpanTypes.HTTP_SERVER
           errored false
           tags {
-            defaultTags()
+            defaultTags(true)
             "$Tags.HTTP_STATUS.key" 200
             "$Tags.HTTP_URL.key" "http://localhost:$port/test"
             "$Tags.HTTP_METHOD.key" "GET"

dd-java-agent/instrumentation/grpc-1.5/src/test/groovy/GrpcStreamingTest.groovy

@@ -97,7 +97,7 @@ class GrpcStreamingTest extends AgentTestRunner {
           tags {
             "$Tags.SPAN_KIND.key" Tags.SPAN_KIND_SERVER
             "$DDTags.SPAN_TYPE" DDSpanTypes.RPC
-            defaultTags()
+            defaultTags(true)
           }
         }
         clientRange.each {

dd-java-agent/instrumentation/grpc-1.5/src/test/groovy/GrpcTest.groovy

@@ -53,7 +53,7 @@ class GrpcTest extends AgentTestRunner {
           tags {
             "$Tags.SPAN_KIND.key" Tags.SPAN_KIND_SERVER
             "$DDTags.SPAN_TYPE" DDSpanTypes.RPC
-            defaultTags()
+            defaultTags(true)
           }
         }
         span(1) {
@@ -144,7 +144,7 @@ class GrpcTest extends AgentTestRunner {
           tags {
             "$Tags.SPAN_KIND.key" Tags.SPAN_KIND_SERVER
             "$DDTags.SPAN_TYPE" DDSpanTypes.RPC
-            defaultTags()
+            defaultTags(true)
           }
         }
         span(1) {
@@ -230,7 +230,7 @@ class GrpcTest extends AgentTestRunner {
             "$Tags.SPAN_KIND.key" Tags.SPAN_KIND_SERVER
             "$DDTags.SPAN_TYPE" DDSpanTypes.RPC
             errorTags error.class, error.message
-            defaultTags()
+            defaultTags(true)
           }
         }
         span(1) {

dd-java-agent/instrumentation/jetty-8/src/test/groovy/JettyHandlerTest.groovy

@@ -1,6 +1,7 @@
 import datadog.trace.agent.test.AgentTestRunner
 import datadog.trace.agent.test.TestUtils
 import datadog.trace.agent.test.utils.OkHttpUtils
+import datadog.trace.api.Config
 import datadog.trace.api.DDSpanTypes
 import okhttp3.OkHttpClient
 import org.eclipse.jetty.continuation.Continuation
@@ -74,8 +75,9 @@ class JettyHandlerTest extends AgentTestRunner {
     tags["http.status_code"] == 200
     tags["thread.name"] != null
     tags["thread.id"] != null
+    tags[Config.RUNTIME_ID_TAG] == Config.get().runtimeId
     tags["span.origin.type"] == handler.class.name
-    tags.size() == 9
+    tags.size() == 10
   }
 
 
@@ -162,10 +164,11 @@ class JettyHandlerTest extends AgentTestRunner {
     tags["http.status_code"] == 500
     tags["thread.name"] != null
     tags["thread.id"] != null
+    tags[Config.RUNTIME_ID_TAG] == Config.get().runtimeId
     tags["span.origin.type"] == handler.class.name
     tags["error"] == true
     tags["error.type"] == RuntimeException.name
     tags["error.stack"] != null
-    tags.size() == 12
+    tags.size() == 13
   }
 }

dd-java-agent/instrumentation/jms/src/latestDepTest/groovy/JMS2Test.groovy

@@ -93,7 +93,7 @@ class JMS2Test extends AgentTestRunner {
           errored false
 
           tags {
-            defaultTags()
+            defaultTags(true)
             "${DDTags.SPAN_TYPE}" DDSpanTypes.MESSAGE_CONSUMER
             "${Tags.COMPONENT.key}" "jms"
             "${Tags.SPAN_KIND.key}" "consumer"
@@ -145,7 +145,7 @@ class JMS2Test extends AgentTestRunner {
           errored false
 
           tags {
-            defaultTags()
+            defaultTags(true)
             "${DDTags.SPAN_TYPE}" DDSpanTypes.MESSAGE_CONSUMER
             "${Tags.COMPONENT.key}" "jms"
             "${Tags.SPAN_KIND.key}" "consumer"

dd-java-agent/instrumentation/jms/src/test/groovy/JMS1Test.groovy

@@ -59,7 +59,7 @@ class JMS1Test extends AgentTestRunner {
           errored false
 
           tags {
-            defaultTags()
+            defaultTags(true)
             "${DDTags.SPAN_TYPE}" DDSpanTypes.MESSAGE_CONSUMER
             "${Tags.COMPONENT.key}" "jms"
             "${Tags.SPAN_KIND.key}" "consumer"
@@ -111,7 +111,7 @@ class JMS1Test extends AgentTestRunner {
           errored false
 
           tags {
-            defaultTags()
+            defaultTags(true)
             "${DDTags.SPAN_TYPE}" DDSpanTypes.MESSAGE_CONSUMER
             "${Tags.COMPONENT.key}" "jms"
             "${Tags.SPAN_KIND.key}" "consumer"

dd-java-agent/instrumentation/kafka-clients-0.11/src/test/groovy/KafkaClientTest.groovy

@@ -1,4 +1,5 @@
 import datadog.trace.agent.test.AgentTestRunner
+import datadog.trace.api.Config
 import org.apache.kafka.clients.consumer.ConsumerRecord
 import org.junit.ClassRule
 import org.springframework.kafka.core.DefaultKafkaConsumerFactory
@@ -93,7 +94,8 @@ class KafkaClientTest extends AgentTestRunner {
     t1tags1["span.type"] == "queue"
     t1tags1["thread.name"] != null
     t1tags1["thread.id"] != null
-    t1tags1.size() == 5
+    t1tags1[Config.RUNTIME_ID_TAG] == Config.get().runtimeId
+    t1tags1.size() == 6
 
     and: // CONSUMER span 0
     def t2span1 = t2[0]
@@ -113,7 +115,8 @@ class KafkaClientTest extends AgentTestRunner {
     t2tags1["offset"] == 0
     t2tags1["thread.name"] != null
     t2tags1["thread.id"] != null
-    t2tags1.size() == 7
+    t2tags1[Config.RUNTIME_ID_TAG] == Config.get().runtimeId
+    t2tags1.size() == 8
 
     def headers = received.headers()
     headers.iterator().hasNext()

dd-java-agent/instrumentation/kafka-streams-0.11/src/test/groovy/KafkaStreamsTest.groovy

@@ -1,4 +1,5 @@
 import datadog.trace.agent.test.AgentTestRunner
+import datadog.trace.api.Config
 import org.apache.kafka.clients.consumer.ConsumerRecord
 import org.apache.kafka.common.serialization.Serdes
 import org.apache.kafka.streams.KafkaStreams
@@ -119,7 +120,8 @@ class KafkaStreamsTest extends AgentTestRunner {
     t1tags1["span.type"] == "queue"
     t1tags1["thread.name"] != null
     t1tags1["thread.id"] != null
-    t1tags1.size() == 5
+    t1tags1[Config.RUNTIME_ID_TAG] == Config.get().runtimeId
+    t1tags1.size() == 6
 
     and: // STREAMING span 0
     def t2span1 = t2[0]
@@ -157,8 +159,9 @@ class KafkaStreamsTest extends AgentTestRunner {
     t2tags2["offset"] == 0
     t2tags2["thread.name"] != null
     t2tags2["thread.id"] != null
+    t2tags2[Config.RUNTIME_ID_TAG] == Config.get().runtimeId
     t2tags2["asdf"] == "testing"
-    t2tags2.size() == 8
+    t2tags2.size() == 9
 
     and: // CONSUMER span 0
     def t3span1 = t3[0]
@@ -178,8 +181,9 @@ class KafkaStreamsTest extends AgentTestRunner {
     t3tags1["offset"] == 0
     t3tags1["thread.name"] != null
     t3tags1["thread.id"] != null
+    t3tags1[Config.RUNTIME_ID_TAG] == Config.get().runtimeId
     t3tags1["testing"] == 123
-    t3tags1.size() == 8
+    t3tags1.size() == 9
 
     def headers = received.headers()
     headers.iterator().hasNext()

dd-java-agent/instrumentation/netty-4.0/src/test/groovy/Netty40ServerTest.groovy

@@ -38,7 +38,12 @@ class Netty40ServerTest extends AgentTestRunner {
     int port = TestUtils.randomOpenPort()
     initializeServer(eventLoopGroup, port, handlers, HttpResponseStatus.OK)
 
-    def request = new Request.Builder().url("http://localhost:$port/").get().build()
+    def request = new Request.Builder()
+      .url("http://localhost:$port/")
+      .header("x-datadog-trace-id", "123")
+      .header("x-datadog-parent-id", "456")
+      .get()
+      .build()
     def response = client.newCall(request).execute()
 
     expect:
@@ -49,6 +54,8 @@ class Netty40ServerTest extends AgentTestRunner {
     assertTraces(1) {
       trace(0, 1) {
         span(0) {
+          traceId "123"
+          parentId "456"
           serviceName "unnamed-java-app"
           operationName "netty.request"
           resourceName "GET /"
@@ -63,7 +70,7 @@ class Netty40ServerTest extends AgentTestRunner {
             "$Tags.PEER_PORT.key" Integer
             "$Tags.SPAN_KIND.key" Tags.SPAN_KIND_SERVER
             "$DDTags.SPAN_TYPE" DDSpanTypes.HTTP_SERVER
-            defaultTags()
+            defaultTags(true)
           }
         }
       }

dd-java-agent/instrumentation/netty-4.1/src/test/groovy/Netty41ServerTest.groovy

@@ -72,7 +72,7 @@ class Netty41ServerTest extends AgentTestRunner {
             "$Tags.PEER_PORT.key" Integer
             "$Tags.SPAN_KIND.key" Tags.SPAN_KIND_SERVER
             "$DDTags.SPAN_TYPE" DDSpanTypes.HTTP_SERVER
-            defaultTags()
+            defaultTags(true)
           }
         }
       }

dd-java-agent/instrumentation/rabbitmq-amqp-2.7/src/test/groovy/RabbitMQTest.groovy

@@ -101,17 +101,17 @@ class RabbitMQTest extends AgentTestRunner {
     and:
     assertTraces(2) {
       trace(0, 1) {
-        rabbitSpan(it, "basic.get <generated>", TEST_WRITER[1][1])
+        rabbitSpan(it, "basic.get <generated>", true, TEST_WRITER[1][1])
       }
       trace(1, 5) {
         span(0) {
           operationName "parent"
         }
         // reverse order
-        rabbitSpan(it, 1, "basic.publish $exchangeName -> $routingKey", span(0))
+        rabbitSpan(it, 1, "basic.publish $exchangeName -> $routingKey", false, span(0))
-        rabbitSpan(it, 2, "queue.bind", span(0))
+        rabbitSpan(it, 2, "queue.bind", false, span(0))
-        rabbitSpan(it, 3, "queue.declare", span(0))
+        rabbitSpan(it, 3, "queue.declare", false, span(0))
-        rabbitSpan(it, 4, "exchange.declare", span(0))
+        rabbitSpan(it, 4, "exchange.declare", false, span(0))
       }
     }
 
@@ -140,7 +140,7 @@ class RabbitMQTest extends AgentTestRunner {
         rabbitSpan(it, "basic.publish <default> -> <generated>")
       }
       trace(2, 1) {
-        rabbitSpan(it, "basic.get <generated>", TEST_WRITER[1][0])
+        rabbitSpan(it, "basic.get <generated>", true, TEST_WRITER[1][0])
       }
     }
   }
@@ -197,7 +197,7 @@ class RabbitMQTest extends AgentTestRunner {
           rabbitSpan(it, "basic.publish $exchangeName -> <all>")
         }
         trace(3 + (it * 2), 1) {
-          rabbitSpan(it, resource, publishSpan)
+          rabbitSpan(it, resource, true, publishSpan)
         }
       }
     }
@@ -220,7 +220,7 @@ class RabbitMQTest extends AgentTestRunner {
 
     assertTraces(1) {
       trace(0, 1) {
-        rabbitSpan(it, command, null, throwable, errorMsg)
+        rabbitSpan(it, command, false, null, throwable, errorMsg)
       }
     }
 
@@ -259,16 +259,31 @@ class RabbitMQTest extends AgentTestRunner {
         rabbitSpan(it, "basic.publish <default> -> some-routing-queue")
       }
       trace(2, 1) {
-        rabbitSpan(it, "basic.get $queue.name", TEST_WRITER[1][0])
+        rabbitSpan(it, "basic.get $queue.name", true, TEST_WRITER[1][0])
       }
     }
   }
 
-  def rabbitSpan(TraceAssert trace, String resource, DDSpan parentSpan = null, Throwable exception = null, String errorMsg = null) {
+  def rabbitSpan(
-    rabbitSpan(trace, 0, resource, parentSpan, exception, errorMsg)
+    TraceAssert trace,
+    String resource,
+    Boolean distributedRootSpan = false,
+    DDSpan parentSpan = null,
+    Throwable exception = null,
+    String errorMsg = null
+  ) {
+    rabbitSpan(trace, 0, resource, distributedRootSpan, parentSpan, exception, errorMsg)
   }
 
-  def rabbitSpan(TraceAssert trace, int index, String resource, DDSpan parentSpan = null, Throwable exception = null, String errorMsg = null) {
+  def rabbitSpan(
+    TraceAssert trace,
+    int index,
+    String resource,
+    Boolean distributedRootSpan = false,
+    DDSpan parentSpan = null,
+    Throwable exception = null,
+    String errorMsg = null
+  ) {
     trace.span(index) {
       serviceName "rabbitmq"
       operationName "amqp.command"
@@ -322,7 +337,7 @@ class RabbitMQTest extends AgentTestRunner {
             "$DDTags.SPAN_TYPE" DDSpanTypes.MESSAGE_CLIENT
             "amqp.command" { it == null || it == resource }
         }
-        defaultTags()
+        defaultTags(distributedRootSpan)
       }
     }
   }

dd-java-agent/instrumentation/servlet-2/src/test/groovy/JettyServlet2Test.groovy

@@ -47,9 +47,6 @@ class JettyServlet2Test extends AgentTestRunner {
 
     jettyServer.setHandler(servletContext)
     jettyServer.start()
-
-    System.out.println(
-      "Jetty server: http://localhost:" + port + "/")
   }
 
   def cleanup() {
@@ -57,11 +54,15 @@ class JettyServlet2Test extends AgentTestRunner {
     jettyServer.destroy()
   }
 
-  def "test #path servlet call"() {
+  def "test #path servlet call (auth: #auth, distributed tracing: #distributedTracing)"() {
     setup:
     def requestBuilder = new Request.Builder()
       .url("http://localhost:$port/ctx/$path")
       .get()
+    if (distributedTracing) {
+      requestBuilder.header("x-datadog-trace-id", "123")
+      requestBuilder.header("x-datadog-parent-id", "456")
+    }
     if (auth) {
       requestBuilder.header(HttpHeaders.AUTHORIZATION, Credentials.basic("user", "password"))
     }
@@ -73,12 +74,17 @@ class JettyServlet2Test extends AgentTestRunner {
     assertTraces(1) {
       trace(0, 1) {
         span(0) {
+          if (distributedTracing) {
+            traceId "123"
+            parentId "456"
+          } else {
+            parent()
+          }
           serviceName "ctx"
           operationName "servlet.request"
           resourceName "GET /ctx/$path"
           spanType DDSpanTypes.WEB_SERVLET
           errored false
-          parent()
           tags {
             "http.url" "http://localhost:$port/ctx/$path"
             "http.method" "GET"
@@ -90,16 +96,18 @@ class JettyServlet2Test extends AgentTestRunner {
             if (auth) {
               "user.principal" "user"
             }
-            defaultTags()
+            defaultTags(distributedTracing)
           }
         }
       }
     }
 
     where:
-    path        | expectedResponse | auth
+    path        | expectedResponse | auth  | distributedTracing
-    "sync"      | "Hello Sync"     | false
+    "sync"      | "Hello Sync"     | false | false
-    "auth/sync" | "Hello Sync"     | true
+    "auth/sync" | "Hello Sync"     | true  | false
+    "sync"      | "Hello Sync"     | false | true
+    "auth/sync" | "Hello Sync"     | true  | true
   }
 
   def "test #path error servlet call"() {

dd-java-agent/instrumentation/servlet-3/src/test/groovy/JettyServlet3Test.groovy

@@ -100,7 +100,7 @@ class JettyServlet3Test extends AgentTestRunner {
             if (auth) {
               "user.principal" "user"
             }
-            defaultTags()
+            defaultTags(distributedTracing)
           }
         }
       }

dd-java-agent/instrumentation/servlet-3/src/test/groovy/TomcatServlet3Test.groovy

@@ -94,7 +94,7 @@ class TomcatServlet3Test extends AgentTestRunner {
             "span.origin.type" ApplicationFilterChain.name
             "servlet.context" "/my-context"
             "http.status_code" 200
-            defaultTags()
+            defaultTags(distributedTracing)
           }
         }
       }

dd-java-agent/instrumentation/sparkjava-2.3/src/test/groovy/SparkJavaBasedTest.groovy

@@ -1,6 +1,7 @@
 import datadog.trace.agent.test.AgentTestRunner
 import datadog.trace.agent.test.TestUtils
 import datadog.trace.agent.test.utils.OkHttpUtils
+import datadog.trace.api.Config
 import datadog.trace.api.DDSpanTypes
 import okhttp3.OkHttpClient
 import okhttp3.Request
@@ -91,8 +92,9 @@ class SparkJavaBasedTest extends AgentTestRunner {
     tags["http.status_code"] == 200
     tags["thread.name"] != null
     tags["thread.id"] != null
+    tags[Config.RUNTIME_ID_TAG] == Config.get().runtimeId
     tags["span.origin.type"] == JettyHandler.name
-    tags.size() == 9
+    tags.size() == 10
   }
 
 }

dd-java-agent/instrumentation/vertx/src/test/groovy/VertxServerTest.groovy

@@ -63,7 +63,7 @@ class VertxServerTest extends AgentTestRunner {
             "$Tags.PEER_PORT.key" Integer
             "$Tags.SPAN_KIND.key" Tags.SPAN_KIND_SERVER
             "$DDTags.SPAN_TYPE" DDSpanTypes.HTTP_SERVER
-            defaultTags()
+            defaultTags(true)
           }
         }
         span(1) {

dd-java-agent/testing/src/main/groovy/datadog/trace/agent/test/asserts/TagsAssert.groovy

@@ -1,14 +1,17 @@
 package datadog.trace.agent.test.asserts
 
 import datadog.opentracing.DDSpan
+import datadog.trace.api.Config
 import groovy.transform.stc.ClosureParams
 import groovy.transform.stc.SimpleType
 
 class TagsAssert {
+  private final String spanParentId
   private final Map<String, Object> tags
   private final Set<String> assertedTags = new TreeSet<>()
 
   private TagsAssert(DDSpan span) {
+    this.spanParentId = span.parentId
     this.tags = span.tags
   }
 
@@ -23,12 +26,21 @@ class TagsAssert {
     asserter.assertTagsAllVerified()
   }
 
-  def defaultTags() {
+  /**
+   * @param distributedRootSpan set to true if current span has a parent span but still considered 'root' for current service
+   */
+  def defaultTags(boolean distributedRootSpan = false) {
     assertedTags.add("thread.name")
     assertedTags.add("thread.id")
+    assertedTags.add(Config.RUNTIME_ID_TAG)
 
     assert tags["thread.name"] != null
     assert tags["thread.id"] != null
+    if ("0" == spanParentId || distributedRootSpan) {
+      assert tags[Config.RUNTIME_ID_TAG] == Config.get().runtimeId
+    } else {
+      assert tags[Config.RUNTIME_ID_TAG] == null
+    }
   }
 
   def errorTags(Class<Throwable> errorType) {

dd-java-agent/testing/src/main/groovy/datadog/trace/agent/test/server/http/TestHttpServer.groovy

@@ -104,7 +104,7 @@ class TestHttpServer implements AutoCloseable {
           childOf(parentSpan)
         }
         tags {
-          defaultTags()
+          defaultTags(parentSpan != null)
         }
       }
     }

dd-java-agent/testing/testing.gradle

@@ -1,7 +1,7 @@
 apply from: "${rootDir}/gradle/java.gradle"
 
 minimumBranchCoverage = 0.5
-minimumInstructionCoverage = 0.6
+minimumInstructionCoverage = 0.5
 excludedClassesConverage += [
   'datadog.trace.agent.test.asserts.*Assert',
   'datadog.trace.agent.test.AgentTestRunner.ErrorCountingListener',

dd-trace-api/src/main/java/datadog/trace/api/Config.java

@@ -6,6 +6,7 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Properties;
+import java.util.UUID;
 import lombok.Getter;
 import lombok.ToString;
 import lombok.extern.slf4j.Slf4j;
@@ -34,7 +35,9 @@ public class Config {
   public static final String PRIORITY_SAMPLING = "priority.sampling";
   public static final String TRACE_RESOLVER_ENABLED = "trace.resolver.enabled";
   public static final String SERVICE_MAPPING = "service.mapping";
+  public static final String GLOBAL_TAGS = "trace.global.tags";
   public static final String SPAN_TAGS = "trace.span.tags";
+  public static final String JMX_TAGS = "trace.jmx.tags";
   public static final String HEADER_TAGS = "trace.header.tags";
   public static final String JMX_FETCH_ENABLED = "jmxfetch.enabled";
   public static final String JMX_FETCH_METRICS_CONFIGS = "jmxfetch.metrics-configs";
@@ -43,6 +46,7 @@ public class Config {
   public static final String JMX_FETCH_STATSD_HOST = "jmxfetch.statsd.host";
   public static final String JMX_FETCH_STATSD_PORT = "jmxfetch.statsd.port";
 
+  public static final String RUNTIME_ID_TAG = "runtime-id";
   public static final String DEFAULT_SERVICE_NAME = "unnamed-java-app";
 
   public static final String DD_AGENT_WRITER_TYPE = "DDAgentWriter";
@@ -58,6 +62,8 @@ public class Config {
 
   public static final int DEFAULT_JMX_FETCH_STATSD_PORT = 8125;
 
+  @Getter private final String runtimeId;
+
   @Getter private final String serviceName;
   @Getter private final String writerType;
   @Getter private final String agentHost;
@@ -65,7 +71,9 @@ public class Config {
   @Getter private final boolean prioritySamplingEnabled;
   @Getter private final boolean traceResolverEnabled;
   @Getter private final Map<String, String> serviceMapping;
-  @Getter private final Map<String, String> spanTags;
+  private final Map<String, String> globalTags;
+  private final Map<String, String> spanTags;
+  private final Map<String, String> jmxTags;
   @Getter private final Map<String, String> headerTags;
   @Getter private final boolean jmxFetchEnabled;
   @Getter private final List<String> jmxFetchMetricsConfigs;
@@ -77,6 +85,8 @@ public class Config {
   // Read order: System Properties -> Env Variables, [-> default value]
   // Visible for testing
   Config() {
+    runtimeId = UUID.randomUUID().toString();
+
     serviceName = getSettingFromEnvironment(SERVICE_NAME, DEFAULT_SERVICE_NAME);
     writerType = getSettingFromEnvironment(WRITER_TYPE, DEFAULT_AGENT_WRITER_TYPE);
     agentHost = getSettingFromEnvironment(AGENT_HOST, DEFAULT_AGENT_HOST);
@@ -86,7 +96,11 @@ public class Config {
     traceResolverEnabled =
         getBooleanSettingFromEnvironment(TRACE_RESOLVER_ENABLED, DEFAULT_TRACE_RESOLVER_ENABLED);
     serviceMapping = getMapSettingFromEnvironment(SERVICE_MAPPING, null);
+
+    globalTags = getMapSettingFromEnvironment(GLOBAL_TAGS, null);
     spanTags = getMapSettingFromEnvironment(SPAN_TAGS, null);
+    jmxTags = getMapSettingFromEnvironment(JMX_TAGS, null);
+
     headerTags = getMapSettingFromEnvironment(HEADER_TAGS, null);
     jmxFetchEnabled =
         getBooleanSettingFromEnvironment(JMX_FETCH_ENABLED, DEFAULT_JMX_FETCH_ENABLED);
@@ -101,6 +115,8 @@ public class Config {
 
   // Read order: Properties -> Parent
   private Config(final Properties properties, final Config parent) {
+    runtimeId = parent.runtimeId;
+
     serviceName = properties.getProperty(SERVICE_NAME, parent.serviceName);
     writerType = properties.getProperty(WRITER_TYPE, parent.writerType);
     agentHost = properties.getProperty(AGENT_HOST, parent.agentHost);
@@ -110,7 +126,11 @@ public class Config {
     traceResolverEnabled =
         getPropertyBooleanValue(properties, TRACE_RESOLVER_ENABLED, parent.traceResolverEnabled);
     serviceMapping = getPropertyMapValue(properties, SERVICE_MAPPING, parent.serviceMapping);
+
+    globalTags = getPropertyMapValue(properties, GLOBAL_TAGS, parent.globalTags);
     spanTags = getPropertyMapValue(properties, SPAN_TAGS, parent.spanTags);
+    jmxTags = getPropertyMapValue(properties, JMX_TAGS, parent.jmxTags);
+
     headerTags = getPropertyMapValue(properties, HEADER_TAGS, parent.headerTags);
     jmxFetchEnabled =
         getPropertyBooleanValue(properties, JMX_FETCH_ENABLED, parent.jmxFetchEnabled);
@@ -126,6 +146,22 @@ public class Config {
         getPropertyIntegerValue(properties, JMX_FETCH_STATSD_PORT, parent.jmxFetchStatsdPort);
   }
 
+  public Map<String, String> getMergedSpanTags() {
+    // DO not include runtimeId into span tags: we only want that added to the root span
+    final Map<String, String> result = newHashMap(globalTags.size() + spanTags.size());
+    result.putAll(globalTags);
+    result.putAll(spanTags);
+    return Collections.unmodifiableMap(result);
+  }
+
+  public Map<String, String> getMergedJmxTags() {
+    final Map<String, String> result = newHashMap(globalTags.size() + jmxTags.size() + 1);
+    result.putAll(globalTags);
+    result.putAll(jmxTags);
+    result.put(RUNTIME_ID_TAG, runtimeId);
+    return Collections.unmodifiableMap(result);
+  }
+
   private static String getSettingFromEnvironment(final String name, final String defaultValue) {
     final String completeName = PREFIX + name;
     final String value =
@@ -195,7 +231,7 @@ public class Config {
     }
 
     final String[] tokens = str.split(",", -1);
-    final Map<String, String> map = new HashMap<>(tokens.length + 1, 1f);
+    final Map<String, String> map = newHashMap(tokens.length);
 
     for (final String token : tokens) {
       final String[] keyValue = token.split(":", -1);
@@ -212,6 +248,10 @@ public class Config {
     return Collections.unmodifiableMap(map);
   }
 
+  private static Map<String, String> newHashMap(final int size) {
+    return new HashMap<>(size + 1, 1f);
+  }
+
   private static List<String> parseList(final String str) {
     if (str == null || str.trim().isEmpty()) {
       return Collections.emptyList();

dd-trace-api/src/test/groovy/datadog/trace/api/ConfigTest.groovy

@@ -5,23 +5,7 @@ import org.junit.contrib.java.lang.system.EnvironmentVariables
 import org.junit.contrib.java.lang.system.RestoreSystemProperties
 import spock.lang.Specification
 
-import static Config.AGENT_HOST
+import static datadog.trace.api.Config.*
-import static Config.AGENT_PORT
-import static Config.HEADER_TAGS
-import static Config.PREFIX
-import static Config.SERVICE_MAPPING
-import static Config.SERVICE_NAME
-import static Config.SPAN_TAGS
-import static Config.WRITER_TYPE
-import static datadog.trace.api.Config.DEFAULT_JMX_FETCH_STATSD_PORT
-import static datadog.trace.api.Config.JMX_FETCH_CHECK_PERIOD
-import static datadog.trace.api.Config.JMX_FETCH_ENABLED
-import static datadog.trace.api.Config.JMX_FETCH_METRICS_CONFIGS
-import static datadog.trace.api.Config.JMX_FETCH_REFRESH_BEANS_PERIOD
-import static datadog.trace.api.Config.JMX_FETCH_STATSD_HOST
-import static datadog.trace.api.Config.JMX_FETCH_STATSD_PORT
-import static datadog.trace.api.Config.PRIORITY_SAMPLING
-import static datadog.trace.api.Config.TRACE_RESOLVER_ENABLED
 
 class ConfigTest extends Specification {
   @Rule
@@ -47,7 +31,8 @@ class ConfigTest extends Specification {
     config.prioritySamplingEnabled == false
     config.traceResolverEnabled == true
     config.serviceMapping == [:]
-    config.spanTags == [:]
+    config.mergedSpanTags == [:]
+    config.mergedJmxTags == [(RUNTIME_ID_TAG): config.getRuntimeId()]
     config.headerTags == [:]
     config.jmxFetchEnabled == false
     config.jmxFetchMetricsConfigs == []
@@ -67,8 +52,10 @@ class ConfigTest extends Specification {
     System.setProperty(PREFIX + PRIORITY_SAMPLING, "true")
     System.setProperty(PREFIX + TRACE_RESOLVER_ENABLED, "false")
     System.setProperty(PREFIX + SERVICE_MAPPING, "a:1")
-    System.setProperty(PREFIX + SPAN_TAGS, "b:2")
+    System.setProperty(PREFIX + GLOBAL_TAGS, "b:2")
-    System.setProperty(PREFIX + HEADER_TAGS, "c:3")
+    System.setProperty(PREFIX + SPAN_TAGS, "c:3")
+    System.setProperty(PREFIX + JMX_TAGS, "d:4")
+    System.setProperty(PREFIX + HEADER_TAGS, "e:5")
     System.setProperty(PREFIX + JMX_FETCH_ENABLED, "true")
     System.setProperty(PREFIX + JMX_FETCH_METRICS_CONFIGS, "/foo.yaml,/bar.yaml")
     System.setProperty(PREFIX + JMX_FETCH_CHECK_PERIOD, "100")
@@ -87,8 +74,9 @@ class ConfigTest extends Specification {
     config.prioritySamplingEnabled == true
     config.traceResolverEnabled == false
     config.serviceMapping == [a: "1"]
-    config.spanTags == [b: "2"]
+    config.mergedSpanTags == [b: "2", c: "3"]
-    config.headerTags == [c: "3"]
+    config.mergedJmxTags == [b: "2", d: "4", (RUNTIME_ID_TAG): config.getRuntimeId()]
+    config.headerTags == [e: "5"]
     config.jmxFetchEnabled == true
     config.jmxFetchMetricsConfigs == ["/foo.yaml", "/bar.yaml"]
     config.jmxFetchCheckPeriod == 100
@@ -140,8 +128,10 @@ class ConfigTest extends Specification {
     properties.setProperty(PRIORITY_SAMPLING, "true")
     properties.setProperty(TRACE_RESOLVER_ENABLED, "false")
     properties.setProperty(SERVICE_MAPPING, "a:1")
-    properties.setProperty(SPAN_TAGS, "b:2")
+    properties.setProperty(GLOBAL_TAGS, "b:2")
-    properties.setProperty(HEADER_TAGS, "c:3")
+    properties.setProperty(SPAN_TAGS, "c:3")
+    properties.setProperty(JMX_TAGS, "d:4")
+    properties.setProperty(HEADER_TAGS, "e:5")
     properties.setProperty(JMX_FETCH_METRICS_CONFIGS, "/foo.yaml,/bar.yaml")
     properties.setProperty(JMX_FETCH_CHECK_PERIOD, "100")
     properties.setProperty(JMX_FETCH_REFRESH_BEANS_PERIOD, "200")
@@ -159,8 +149,9 @@ class ConfigTest extends Specification {
     config.prioritySamplingEnabled == true
     config.traceResolverEnabled == false
     config.serviceMapping == [a: "1"]
-    config.spanTags == [b: "2"]
+    config.mergedSpanTags == [b: "2", c: "3"]
-    config.headerTags == [c: "3"]
+    config.mergedJmxTags == [b: "2", d: "4", (RUNTIME_ID_TAG): config.getRuntimeId()]
+    config.headerTags == [e: "5"]
     config.jmxFetchMetricsConfigs == ["/foo.yaml", "/bar.yaml"]
     config.jmxFetchCheckPeriod == 100
     config.jmxFetchRefreshBeansPeriod == 200

dd-trace-ot/src/main/java/datadog/opentracing/DDTracer.java

@@ -52,6 +52,8 @@ public class DDTracer implements io.opentracing.Tracer, Closeable, datadog.trace
   /** Scope manager is in charge of managing the scopes from which spans are created */
   final ContextualScopeManager scopeManager = new ContextualScopeManager();
 
+  /** Value for runtime-id tag */
+  private final String runtimeId;
   /** A set of tags that are added to every span */
   private final Map<String, String> defaultSpanTags;
   /** A configured mapping of service names to update with new values */
@@ -95,17 +97,24 @@ public class DDTracer implements io.opentracing.Tracer, Closeable, datadog.trace
         serviceName,
         Writer.Builder.forConfig(config),
         Sampler.Builder.forConfig(config),
-        config.getSpanTags(),
+        config.getRuntimeId(),
+        config.getMergedSpanTags(),
         config.getServiceMapping(),
         config.getHeaderTags());
     log.debug("Using config: {}", config);
   }
 
-  public DDTracer(final String serviceName, final Writer writer, final Sampler sampler) {
+  /** Visible for testing */
+  DDTracer(
+      final String serviceName,
+      final Writer writer,
+      final Sampler sampler,
+      final String runtimeId) {
     this(
         serviceName,
         writer,
         sampler,
+        runtimeId,
         Collections.<String, String>emptyMap(),
         Collections.<String, String>emptyMap(),
         Collections.<String, String>emptyMap());
@@ -120,7 +129,8 @@ public class DDTracer implements io.opentracing.Tracer, Closeable, datadog.trace
         config.getServiceName(),
         writer,
         Sampler.Builder.forConfig(config),
-        config.getSpanTags(),
+        config.getRuntimeId(),
+        config.getMergedSpanTags(),
         config.getServiceMapping(),
         config.getHeaderTags());
   }
@@ -129,9 +139,11 @@ public class DDTracer implements io.opentracing.Tracer, Closeable, datadog.trace
       final String serviceName,
       final Writer writer,
       final Sampler sampler,
+      final String runtimeId,
       final Map<String, String> defaultSpanTags,
       final Map<String, String> serviceNameMappings,
       final Map<String, String> taggedHeaders) {
+    assert runtimeId != null;
     assert defaultSpanTags != null;
     assert serviceNameMappings != null;
     assert taggedHeaders != null;
@@ -141,6 +153,7 @@ public class DDTracer implements io.opentracing.Tracer, Closeable, datadog.trace
     this.writer.start();
     this.sampler = sampler;
     this.defaultSpanTags = defaultSpanTags;
+    this.runtimeId = runtimeId;
     this.serviceNameMappings = serviceNameMappings;
 
     try {
@@ -335,6 +348,8 @@ public class DDTracer implements io.opentracing.Tracer, Closeable, datadog.trace
         + writer
         + ", sampler="
         + sampler
+        + ", runtimeId="
+        + runtimeId
         + ", defaultSpanTags="
         + defaultSpanTags
         + '}';
@@ -361,7 +376,7 @@ public class DDTracer implements io.opentracing.Tracer, Closeable, datadog.trace
     private final String operationName;
 
     // Builder attributes
-    private Map<String, Object> tags = new HashMap<String, Object>(defaultSpanTags);
+    private final Map<String, Object> tags = new HashMap<String, Object>(defaultSpanTags);
     private long timestampMicro;
     private SpanContext parent;
     private String serviceName;
@@ -536,12 +551,8 @@ public class DDTracer implements io.opentracing.Tracer, Closeable, datadog.trace
         traceId = ddsc.getTraceId();
         parentSpanId = ddsc.getSpanId();
         baggage = ddsc.getBaggage();
-        if (tags.isEmpty() && !ddsc.getTags().isEmpty()) {
+        tags.putAll(ddsc.getTags());
-          tags = new HashMap<>();
+        tags.put(Config.RUNTIME_ID_TAG, runtimeId);
-        }
-        if (!ddsc.getTags().isEmpty()) {
-          tags.putAll(ddsc.getTags());
-        }
         parentTrace = new PendingTrace(DDTracer.this, traceId, serviceNameMappings);
         samplingPriority = ddsc.getSamplingPriority();
 
@@ -550,6 +561,7 @@ public class DDTracer implements io.opentracing.Tracer, Closeable, datadog.trace
         traceId = generateNewId();
         parentSpanId = "0";
         baggage = null;
+        tags.put(Config.RUNTIME_ID_TAG, runtimeId);
         parentTrace = new PendingTrace(DDTracer.this, traceId, serviceNameMappings);
         samplingPriority = PrioritySampling.UNSET;
       }

dd-trace-ot/src/test/groovy/datadog/opentracing/DDSpanBuilderTest.groovy

@@ -12,6 +12,7 @@ import static org.mockito.Mockito.when
 
 class DDSpanBuilderTest extends Specification {
   def writer = new ListWriter()
+  def config = Config.get()
   def tracer = new DDTracer(writer)
 
   def "build simple span"() {
@@ -51,8 +52,9 @@ class DDSpanBuilderTest extends Specification {
 
     then:
     span.getTags() == [
-      (DDTags.THREAD_NAME): Thread.currentThread().getName(),
+      (DDTags.THREAD_NAME)   : Thread.currentThread().getName(),
-      (DDTags.THREAD_ID)  : Thread.currentThread().getId(),
+      (DDTags.THREAD_ID)     : Thread.currentThread().getId(),
+      (Config.RUNTIME_ID_TAG): config.getRuntimeId()
     ]
 
     when:
@@ -263,7 +265,7 @@ class DDSpanBuilderTest extends Specification {
     span.parentId == extractedContext.spanId
     span.samplingPriority == extractedContext.samplingPriority
     span.context().baggageItems == extractedContext.baggage
-    span.context().@tags == extractedContext.tags
+    span.context().@tags == extractedContext.tags + [(Config.RUNTIME_ID_TAG): config.getRuntimeId()]
 
     where:
     extractedContext                                                      | _
@@ -274,15 +276,16 @@ class DDSpanBuilderTest extends Specification {
   def "global span tags populated on each span"() {
     setup:
     System.setProperty("dd.trace.span.tags", tagString)
-    tracer = new DDTracer(new Config(), writer)
+    def config = new Config()
+    tracer = new DDTracer(config, writer)
     def span = tracer.buildSpan("op name").withServiceName("foo").start()
-    tags.putAll([
-      (DDTags.THREAD_NAME): Thread.currentThread().getName(),
-      (DDTags.THREAD_ID)  : Thread.currentThread().getId(),
-    ])
 
     expect:
-    span.tags == tags
+    span.tags == tags + [
+      (DDTags.THREAD_NAME)   : Thread.currentThread().getName(),
+      (DDTags.THREAD_ID)     : Thread.currentThread().getId(),
+      (Config.RUNTIME_ID_TAG): config.getRuntimeId()
+    ]
 
     cleanup:
     System.clearProperty("dd.trace.span.tags")

dd-trace-ot/src/test/groovy/datadog/opentracing/DDSpanTest.groovy

@@ -11,7 +11,7 @@ import static datadog.trace.api.Config.DEFAULT_SERVICE_NAME
 
 class DDSpanTest extends Specification {
   def writer = new ListWriter()
-  def tracer = new DDTracer(DEFAULT_SERVICE_NAME, writer, new RateByServiceSampler())
+  def tracer = new DDTracer(DEFAULT_SERVICE_NAME, writer, new RateByServiceSampler(), "some-runtime-id")
 
   def "getters and setters"() {
     setup:

dd-trace-ot/src/test/groovy/datadog/opentracing/TraceInterceptorTest.groovy

@@ -142,7 +142,8 @@ class TraceInterceptorTest extends Specification {
     tags["span.type"] == "modifiedST-null"
     tags["thread.name"] != null
     tags["thread.id"] != null
-    tags.size() == 6
+    tags["runtime-id"] != null
+    tags.size() == 7
   }
 
   def "register interceptor through bridge"() {

dd-trace-ot/src/test/groovy/datadog/opentracing/decorators/SpanDecoratorTest.groovy

@@ -42,7 +42,15 @@ class SpanDecoratorTest extends Specification {
 
   def "set service name"() {
     setup:
-    tracer = new DDTracer("wrong-service", new LoggingWriter(), new AllSampler(), emptyMap(), mapping, emptyMap())
+    tracer = new DDTracer(
+      "wrong-service",
+      new LoggingWriter(),
+      new AllSampler(),
+      "some-runtime-id",
+      emptyMap(),
+      mapping,
+      emptyMap()
+    )
 
     when:
     def span = tracer.buildSpan("some span").withTag(tag, name).start()
@@ -65,7 +73,15 @@ class SpanDecoratorTest extends Specification {
 
   def "default or configured service name can be remapped without setting tag"() {
     setup:
-    tracer = new DDTracer(serviceName, new LoggingWriter(), new AllSampler(), emptyMap(), mapping, emptyMap())
+    tracer = new DDTracer(
+      serviceName,
+      new LoggingWriter(),
+      new AllSampler(),
+      "some-runtime-id",
+      emptyMap(),
+      mapping,
+      emptyMap()
+    )
 
     when:
     def span = tracer.buildSpan("some span").start()
@@ -103,7 +119,15 @@ class SpanDecoratorTest extends Specification {
 
   def "set service name from servlet.context with context '#context' for service #serviceName"() {
     setup:
-    tracer = new DDTracer(serviceName, new LoggingWriter(), new AllSampler(), emptyMap(), mapping, emptyMap())
+    tracer = new DDTracer(
+      serviceName,
+      new LoggingWriter(),
+      new AllSampler(),
+      "some-runtime-id",
+      emptyMap(),
+      mapping,
+      emptyMap()
+    )
 
     when:
     def span = tracer.buildSpan("some span").start()

dd-trace-ot/src/test/groovy/datadog/trace/DDTracerTest.groovy

@@ -64,7 +64,8 @@ class DDTracerTest extends Specification {
     System.setProperty(PREFIX + HEADER_TAGS, mapString)
 
     when:
-    def tracer = new DDTracer(new Config())
+    def config = new Config()
+    def tracer = new DDTracer(config)
     def taggedHeaders = tracer.registry.codecs.values().first().taggedHeaders
 
     then: