Use jmx AppConfig Builder instead of factory method

Expose new options for configuring JMXFetch with standard datadog-agent config files with `jvm_direct: true` set as an instance attribute (this will be ignored by the datadog-agent).

For Example:
* `dd.agent.conf.d=/opt/datadog-agent/etc/conf.d`
* `dd.jmxfetch.configs=activemq.d/conf.yaml,jmx.d/conf.yaml`
will load jmx configs in those two files that have `jvm_direct: true` in their `instance` setup.

Environment variables can also be used: `DD_AGENT_CONF_D` and `DD_JMXFETCH_CONFIGS`

Depends on https://github.com/DataDog/jmxfetch/releases/tag/0.29.0 being released.

dd-java-agent/agent-jmxfetch/agent-jmxfetch.gradle

@@ -4,7 +4,10 @@ plugins {
 apply from: "${rootDir}/gradle/java.gradle"
 
 dependencies {
-  compile 'com.datadoghq:jmxfetch:0.27.0'
+  compile('com.datadoghq:jmxfetch:0.29.0'){
+    exclude group: 'org.slf4j', module: 'slf4j-log4j12'
+    exclude group: 'log4j', module: 'log4j'
+  }
   compile deps.slf4j
   compile project(':dd-trace-api')
 }
@@ -32,12 +35,16 @@ tasks.register("submodulesUpdate", Exec) {
   group 'Build Setup'
   description 'Initializes and updates integrations-core git submodule'
   commandLine 'git', 'submodule', 'update', '--init', 'integrations-core'
+  inputs.file file("${project.rootDir}/.git/modules/dd-java-agent/agent-jmxfetch/integrations-core/HEAD")
+  outputs.dir file("$projectDir/integrations-core")
 }
 
 tasks.register("copyMetricConfigs", Exec) {
   group 'Build Setup'
   description 'Copy metrics.yaml files from integrations-core into resources'
   commandLine './copy-metric-configs.sh', 'integrations-core', sourceSets.main.output.resourcesDir
+  inputs.dir file("$projectDir/integrations-core")
+  outputs.dir sourceSets.main.output.resourcesDir
   doFirst {
     // Ensure the resources directory is available.
     file(sourceSets.main.output.resourcesDir).mkdirs()

dd-java-agent/agent-jmxfetch/src/main/java/datadog/trace/agent/jmxfetch/JMXFetch.java

@@ -1,5 +1,7 @@
 package datadog.trace.agent.jmxfetch;
 
+import static org.datadog.jmxfetch.AppConfig.ACTION_COLLECT;
+
 import com.google.common.collect.ImmutableList;
 import datadog.trace.api.Config;
 import java.io.IOException;
@@ -16,6 +18,7 @@ import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.io.IOUtils;
 import org.datadog.jmxfetch.App;
 import org.datadog.jmxfetch.AppConfig;
+import org.datadog.jmxfetch.reporter.ReporterFactory;
 
 @Slf4j
 public class JMXFetch {
@@ -38,6 +41,14 @@ public class JMXFetch {
       return;
     }
 
+    if (!log.isDebugEnabled()
+        && System.getProperty("org.slf4j.simpleLogger.log.org.datadog.jmxfetch") == null) {
+      // Reduce noisiness of jmxfetch logging.
+      System.setProperty("org.slf4j.simpleLogger.log.org.datadog.jmxfetch", "warn");
+    }
+
+    final String agentConfDPath = config.getAgentConfDPath();
+    final List<String> jmxFetchConfigs = config.getJmxFetchConfigs();
     final List<String> internalMetricsConfigs = getInternalMetricFiles();
     final List<String> metricsConfigs = config.getJmxFetchMetricsConfigs();
     final Integer checkPeriod = config.getJmxFetchCheckPeriod();
@@ -48,7 +59,9 @@ public class JMXFetch {
     final String logLevel = getLogLevel();
 
     log.info(
-        "JMXFetch config: {} {} {} {} {} {} {} {}",
+        "JMXFetch config: {} {} {} {} {} {} {} {} {} {}",
+        agentConfDPath,
+        jmxFetchConfigs,
         internalMetricsConfigs,
         metricsConfigs,
         checkPeriod,
@@ -57,17 +70,24 @@ public class JMXFetch {
         reporter,
         logLocation,
         logLevel);
-    final AppConfig appConfig =
-        AppConfig.create(
-            DEFAULT_CONFIGS,
-            internalMetricsConfigs,
-            metricsConfigs,
-            checkPeriod,
-            refreshBeansPeriod,
-            globalTags,
-            reporter,
-            logLocation,
-            logLevel);
+
+    final AppConfig.AppConfigBuilder configBuilder =
+        AppConfig.builder()
+            .action(ImmutableList.of(ACTION_COLLECT))
+            .confdDirectory(agentConfDPath)
+            .yamlFileList(jmxFetchConfigs)
+            .targetDirectInstances(true)
+            .instanceConfigResources(DEFAULT_CONFIGS)
+            .metricConfigResources(internalMetricsConfigs)
+            .metricConfigFiles(metricsConfigs)
+            .refreshBeansPeriod(refreshBeansPeriod)
+            .globalTags(globalTags)
+            .reporter(ReporterFactory.getReporter(reporter));
+
+    if (checkPeriod != null) {
+      configBuilder.checkPeriod(checkPeriod);
+    }
+    final AppConfig appConfig = configBuilder.build();
 
     final Thread thread =
         new Thread(

dd-java-agent/agent-jmxfetch/src/main/resources/jmxfetch-config.yaml

@@ -3,6 +3,6 @@ init_config:
   new_gc_metrics: true
 
 instances:
-- jmx_url: service:jmx:local:///
-  conf:
-    # Intentionally left empty for now
+  - jvm_direct: true
+    name: dd-java-agent default
+    conf: [] # Intentionally left empty for now

dd-trace-api/src/main/java/datadog/trace/api/Config.java

@@ -64,7 +64,9 @@ public class Config {
   public static final String PROPAGATION_STYLE_EXTRACT = "propagation.style.extract";
   public static final String PROPAGATION_STYLE_INJECT = "propagation.style.inject";
 
+  public static final String AGENT_CONF_D = "agent.conf.d";
   public static final String JMX_FETCH_ENABLED = "jmxfetch.enabled";
+  public static final String JMX_FETCH_CONFIGS = "jmxfetch.configs";
   public static final String JMX_FETCH_METRICS_CONFIGS = "jmxfetch.metrics-configs";
   public static final String JMX_FETCH_CHECK_PERIOD = "jmxfetch.check-period";
   public static final String JMX_FETCH_REFRESH_BEANS_PERIOD = "jmxfetch.refresh-beans-period";
@@ -146,8 +148,10 @@ public class Config {
   @Getter private final Set<PropagationStyle> propagationStylesToExtract;
   @Getter private final Set<PropagationStyle> propagationStylesToInject;
 
+  @Getter private final String agentConfDPath;
   @Getter private final boolean jmxFetchEnabled;
-  @Getter private final List<String> jmxFetchMetricsConfigs;
+  @Getter private final List<String> jmxFetchConfigs;
+  @Deprecated @Getter private final List<String> jmxFetchMetricsConfigs;
   @Getter private final Integer jmxFetchCheckPeriod;
   @Getter private final Integer jmxFetchRefreshBeansPeriod;
   @Getter private final String jmxFetchStatsdHost;
@@ -218,8 +222,10 @@ public class Config {
             PropagationStyle.class,
             true);
 
+    agentConfDPath = getSettingFromEnvironment(AGENT_CONF_D, null);
     jmxFetchEnabled =
         getBooleanSettingFromEnvironment(JMX_FETCH_ENABLED, DEFAULT_JMX_FETCH_ENABLED);
+    jmxFetchConfigs = getListSettingFromEnvironment(JMX_FETCH_CONFIGS, null);
     jmxFetchMetricsConfigs = getListSettingFromEnvironment(JMX_FETCH_METRICS_CONFIGS, null);
     jmxFetchCheckPeriod = getIntegerSettingFromEnvironment(JMX_FETCH_CHECK_PERIOD, null);
     jmxFetchRefreshBeansPeriod =
@@ -298,8 +304,10 @@ public class Config {
             ? parent.propagationStylesToInject
             : parsedPropagationStylesToInject;
 
+    agentConfDPath = properties.getProperty(AGENT_CONF_D, parent.agentConfDPath);
     jmxFetchEnabled =
         getPropertyBooleanValue(properties, JMX_FETCH_ENABLED, parent.jmxFetchEnabled);
+    jmxFetchConfigs = getPropertyListValue(properties, JMX_FETCH_CONFIGS, parent.jmxFetchConfigs);
     jmxFetchMetricsConfigs =
         getPropertyListValue(properties, JMX_FETCH_METRICS_CONFIGS, parent.jmxFetchMetricsConfigs);
     jmxFetchCheckPeriod =