Merge pull request #639 from DataDog/tyler/jmxfetch

Embedding existing jmx integrations with git submodule

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "dd-java-agent/agent-jmxfetch/integrations-core"]
+	path = dd-java-agent/agent-jmxfetch/integrations-core
+	url = https://github.com/DataDog/integrations-core.git

dd-java-agent/agent-jmxfetch/agent-jmxfetch.gradle

@@ -4,7 +4,7 @@ plugins {
 apply from: "${rootDir}/gradle/java.gradle"
 
 dependencies {
-  compile 'com.datadoghq:jmxfetch:0.24.0'
+  compile 'com.datadoghq:jmxfetch:0.25.0'
   compile deps.slf4j
   compile project(':dd-trace-api')
 }
@@ -27,3 +27,23 @@ shadowJar {
 jar {
   classifier = 'unbundled'
 }
+
+task submodulesUpdate(type:Exec) {
+  group 'Build Setup'
+  description 'Initializes and updates integrations-core git submodule'
+  commandLine 'git', 'submodule', 'update', '--init', 'integrations-core'
+}
+
+task copyMetricConfigs(type:Exec) {
+  group 'Build Setup'
+  description 'Copy metrics.yaml files from integrations-core into resources'
+  commandLine './copy-metric-configs.sh', 'integrations-core', sourceSets.main.output.resourcesDir
+  doFirst {
+    // Ensure the resources directory is available.
+    file(sourceSets.main.output.resourcesDir).mkdirs()
+  }
+}
+
+processResources.dependsOn copyMetricConfigs
+copyMetricConfigs.dependsOn submodulesUpdate
+// Runs the copy task before processResources, so duplicate files in the local repo would overwrite imported ones.

dd-java-agent/agent-jmxfetch/copy-metric-configs.sh

@@ -0,0 +1,51 @@
+#!/usr/bin/env bash
+
+# Debug mode:
+# set -x
+
+function print_usage() {
+  echo "Usage: $0 search_directory build_resources_output_directory"
+}
+
+search_directory=$1
+if [ ! -d "$search_directory" ]; then
+  echo "Must specify a valid search_directory"
+  print_usage
+  exit 1
+fi
+
+build_resources_output_directory=$2
+if [ ! -d "$build_resources_output_directory" ]; then
+  echo "Must specify a valid build_resources_output_directory"
+  print_usage
+  exit 1
+fi
+# Add the full package path.
+build_resources_output_directory="$build_resources_output_directory/datadog/trace/agent/jmxfetch"
+
+# Find all the metrics.yaml files containing "jmx_metrics:"
+metrics_files=$(grep --include=metrics.yaml -rwl $search_directory -e 'jmx_metrics:')
+
+if [ -z "$metrics_files" ]; then
+  echo "No metrics.yaml files with jmx_metrics blocks found."
+  print_usage
+  exit 1
+fi
+
+# reset file and ensure directories exists
+> $build_resources_output_directory/metricconfigs.txt
+mkdir -p $build_resources_output_directory/metricconfigs
+
+for input_file in $metrics_files ; do
+  # generate new name based on integration folder name which should look like this:
+  # integrations-core/<integration_name>/datadog_checks/<integration_name>/data/metrics.yaml
+  output_file=$(echo "$input_file" | awk -F/ '{print $2}')
+
+  # save file name in metricconfigs.txt
+  echo "$output_file.yaml" >> $build_resources_output_directory/metricconfigs.txt
+
+  # copy to output location
+  output_file="$build_resources_output_directory/metricconfigs/$output_file.yaml"
+  cp $input_file $output_file
+done
+

dd-java-agent/agent-jmxfetch/integrations-core

@@ -0,0 +1 @@
+Subproject commit 3e38b4e75edcee3ca84f022ea50240b0fc0537f2

dd-java-agent/agent-jmxfetch/src/main/java/datadog/trace/agent/jmxfetch/JMXFetch.java

@@ -2,9 +2,16 @@ package datadog.trace.agent.jmxfetch;
 
 import com.google.common.collect.ImmutableList;
 import datadog.trace.api.Config;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URL;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.io.IOUtils;
 import org.datadog.jmxfetch.App;
 import org.datadog.jmxfetch.AppConfig;
 
@@ -27,6 +34,7 @@ public class JMXFetch {
       return;
     }
 
+    final List<String> internalMetricsConfigs = getInternalMetricFiles();
     final List<String> metricsConfigs = config.getJmxFetchMetricsConfigs();
     final Integer checkPeriod = config.getJmxFetchCheckPeriod();
     final Integer refreshBeansPeriod = config.getJmxFetchRefreshBeansPeriod();
@@ -36,7 +44,8 @@ public class JMXFetch {
     final String logLevel = getLogLevel();
 
     log.error(
-        "JMXFetch config: {} {} {} {} {} {} {}",
+        "JMXFetch config: {} {} {} {} {} {} {} {}",
+        internalMetricsConfigs,
         metricsConfigs,
         checkPeriod,
         refreshBeansPeriod,
@@ -47,6 +56,7 @@ public class JMXFetch {
     final AppConfig appConfig =
         AppConfig.create(
             DEFAULT_CONFIGS,
+            internalMetricsConfigs,
             metricsConfigs,
             checkPeriod,
             refreshBeansPeriod,
@@ -95,6 +105,29 @@ public class JMXFetch {
     return "statsd:" + host + ":" + config.getJmxFetchStatsdPort();
   }
 
+  private static List<String> getInternalMetricFiles() {
+    try {
+      final InputStream metricConfigsStream =
+          JMXFetch.class.getResourceAsStream("metricconfigs.txt");
+      if (metricConfigsStream == null) {
+        log.debug("metricconfigs not found. returning empty set");
+        return Collections.emptyList();
+      } else {
+        final String configs = IOUtils.toString(metricConfigsStream, StandardCharsets.UTF_8);
+        final String[] split = configs.split("\n");
+        final List<String> result = new ArrayList<>(split.length);
+        for (final String config : split) {
+          final URL resource = JMXFetch.class.getResource("metricconfigs/" + config);
+          result.add(resource.getPath().split("\\.jar!/")[1]);
+        }
+        return result;
+      }
+    } catch (final IOException e) {
+      log.debug("error reading metricconfigs. returning empty set", e);
+      return Collections.emptyList();
+    }
+  }
+
   private static String getLogLocation() {
     return System.getProperty("org.slf4j.simpleLogger.logFile", "System.err");
   }

dd-java-agent/agent-jmxfetch/src/main/resources/datadog/trace/agent/jmxfetch/README

@@ -0,0 +1,8 @@
+# Metric Configs
+
+Files from [integrations-core](https://github.com/DataDog/integrations-core/search?q=jmx_metrics&unscoped_q=jmx_metrics)
+are copied here at build time by the `copy-metric-configs.sh` script which is called by gradle after initializing the submodule.
+
+These are then bundled in `dd-java-agent.jar`. Due to limitations in Java, it is non-trivial
+to get all these files from within the jar without knowing their names.
+Consequently, we list out each integration in `metricconfigs.txt` so the agent can reference them.