Embed exporters into agent jar (#491)

* Embed all exporters into agent jar

* Publish agent with exporters and without

* Polish

agent-bootstrap/src/main/java/io/opentelemetry/auto/config/Config.java

@@ -54,6 +54,7 @@ public class Config {
   private static final Pattern ENV_REPLACEMENT = Pattern.compile("[^a-zA-Z0-9_]");
 
   public static final String EXPORTER_JAR = "exporter.jar";
+  public static final String EXPORTER = "exporter";
   public static final String PROPAGATORS = "propagators";
   public static final String CONFIGURATION_FILE = "trace.config";
   public static final String TRACE_ENABLED = "trace.enabled";
@@ -106,6 +107,7 @@ public class Config {
   public static final boolean DEFAULT_SQL_NORMALIZER_ENABLED = true;
 
   @Getter private final String exporterJar;
+  @Getter private final String exporter;
   @Getter private final List<String> propagators;
   @Getter private final boolean traceEnabled;
   @Getter private final boolean integrationsEnabled;
@@ -157,6 +159,7 @@ public class Config {
 
     propagators = getListSettingFromEnvironment(PROPAGATORS, null);
     exporterJar = getSettingFromEnvironment(EXPORTER_JAR, null);
+    exporter = getSettingFromEnvironment(EXPORTER, "otlp");
     traceEnabled = getBooleanSettingFromEnvironment(TRACE_ENABLED, DEFAULT_TRACE_ENABLED);
     integrationsEnabled =
         getBooleanSettingFromEnvironment(INTEGRATIONS_ENABLED, DEFAULT_INTEGRATIONS_ENABLED);
@@ -218,6 +221,7 @@ public class Config {
   // Read order: Properties -> Parent
   private Config(final Properties properties, final Config parent) {
     exporterJar = properties.getProperty(EXPORTER_JAR, parent.exporterJar);
+    exporter = properties.getProperty(EXPORTER, parent.exporter);
 
     propagators = getPropertyListValue(properties, PROPAGATORS, parent.propagators);
 

agent-tooling/src/main/java/io/opentelemetry/auto/tooling/TracerInstaller.java

@@ -15,7 +15,6 @@
  */
 package io.opentelemetry.auto.tooling;
 
-import com.google.common.annotations.VisibleForTesting;
 import io.opentelemetry.auto.config.Config;
 import io.opentelemetry.sdk.OpenTelemetrySdk;
 import io.opentelemetry.sdk.contrib.auto.config.MetricExporterFactory;
@@ -36,16 +35,18 @@ import lombok.extern.slf4j.Slf4j;
 @Slf4j
 public class TracerInstaller {
   /** Register agent tracer if no agent tracer is already registered. */
+  @SuppressWarnings("unused")
   public static synchronized void installAgentTracer() {
     if (Config.get().isTraceEnabled()) {
 
       configure();
-      // Try to create an exporter
+      // Try to create an exporter from external jar file
       final String exporterJar = Config.get().getExporterJar();
       if (exporterJar != null) {
         installExportersFromJar(exporterJar);
       } else {
-        log.warn("No exporter is specified. Tracing will run but spans are dropped");
+        // Try to create embedded exporter
+        installExporters(Config.get().getExporter());
       }
     } else {
       log.info("Tracing is disabled.");
@@ -54,7 +55,33 @@ public class TracerInstaller {
     PropagatorsInitializer.initializePropagators(Config.get().getPropagators());
   }
 
-  @VisibleForTesting
+  private static synchronized void installExporters(final String exporterName) {
+    final SpanExporterFactory spanExporterFactory = findSpanExporterFactory(exporterName);
+    if (spanExporterFactory != null) {
+      final DefaultExporterConfig config = new DefaultExporterConfig("exporter");
+      installExporter(spanExporterFactory, config);
+    } else {
+      log.warn("No {} span exporter found", exporterName);
+      log.warn("No valid span exporter found. Tracing will run but spans are dropped");
+    }
+  }
+
+  private static SpanExporterFactory findSpanExporterFactory(String exporterName) {
+    final ServiceLoader<SpanExporterFactory> serviceLoader =
+        ServiceLoader.load(SpanExporterFactory.class, TracerInstaller.class.getClassLoader());
+
+    for (SpanExporterFactory spanExporterFactory : serviceLoader) {
+      if (spanExporterFactory
+          .getClass()
+          .getSimpleName()
+          .toLowerCase()
+          .startsWith(exporterName.toLowerCase())) {
+        return spanExporterFactory;
+      }
+    }
+    return null;
+  }
+
   private static synchronized void installExportersFromJar(final String exporterJar) {
     final URL url;
     try {
@@ -70,33 +97,44 @@ public class TracerInstaller {
 
     final SpanExporterFactory spanExporterFactory =
         getExporterFactory(SpanExporterFactory.class, exporterLoader);
+
     if (spanExporterFactory != null) {
-      final SpanExporter spanExporter = spanExporterFactory.fromConfig(config);
+      installExporter(spanExporterFactory, config);
-      final BatchSpanProcessor spanProcessor =
-          BatchSpanProcessor.newBuilder(spanExporter)
-              .readEnvironmentVariables()
-              .readSystemProperties()
-              .build();
-      OpenTelemetrySdk.getTracerProvider().addSpanProcessor(spanProcessor);
-      log.info("Installed span exporter: " + spanExporter.getClass().getName());
     } else {
-      log.warn("No matching providers in jar " + exporterJar);
+      log.warn("No span exporter found in {}", exporterJar);
       log.warn("No valid exporter found. Tracing will run but spans are dropped");
     }
 
     final MetricExporterFactory metricExporterFactory =
         getExporterFactory(MetricExporterFactory.class, exporterLoader);
     if (metricExporterFactory != null) {
-      final MetricExporter metricExporter = metricExporterFactory.fromConfig(config);
+      installExporter(metricExporterFactory, config);
-      IntervalMetricReader.builder()
-          .setMetricExporter(metricExporter)
-          .setMetricProducers(
-              Collections.singleton(OpenTelemetrySdk.getMeterProvider().getMetricProducer()))
-          .build();
-      log.info("Installed metric exporter: " + metricExporter.getClass().getName());
     }
   }
 
+  private static void installExporter(
+      MetricExporterFactory metricExporterFactory, DefaultExporterConfig config) {
+    final MetricExporter metricExporter = metricExporterFactory.fromConfig(config);
+    IntervalMetricReader.builder()
+        .setMetricExporter(metricExporter)
+        .setMetricProducers(
+            Collections.singleton(OpenTelemetrySdk.getMeterProvider().getMetricProducer()))
+        .build();
+    log.info("Installed metric exporter: " + metricExporter.getClass().getName());
+  }
+
+  private static void installExporter(
+      SpanExporterFactory spanExporterFactory, DefaultExporterConfig config) {
+    final SpanExporter spanExporter = spanExporterFactory.fromConfig(config);
+    final BatchSpanProcessor spanProcessor =
+        BatchSpanProcessor.newBuilder(spanExporter)
+            .readEnvironmentVariables()
+            .readSystemProperties()
+            .build();
+    OpenTelemetrySdk.getTracerProvider().addSpanProcessor(spanProcessor);
+    log.info("Installed span exporter: " + spanExporter.getClass().getName());
+  }
+
   private static <F> F getExporterFactory(
       final Class<F> service, final ExporterClassLoader exporterLoader) {
     final ServiceLoader<F> serviceLoader = ServiceLoader.load(service, exporterLoader);
@@ -114,7 +152,7 @@ public class TracerInstaller {
   }
 
   private static void configure() {
-    /** Update trace config from env vars or sys props */
+    /* Update trace config from env vars or sys props */
     final TraceConfig activeTraceConfig =
         OpenTelemetrySdk.getTracerProvider().getActiveTraceConfig();
     OpenTelemetrySdk.getTracerProvider()
@@ -126,6 +164,7 @@ public class TracerInstaller {
                 .build());
   }
 
+  @SuppressWarnings("unused")
   public static void logVersionInfo() {
     VersionLogger.logAllVersions();
     log.debug(

auto-exporters/auto-exporters.gradle

@@ -1,3 +1,7 @@
+plugins {
+  id "com.github.johnrengelman.shadow"
+}
+
 apply from: "${rootDir}/gradle/java.gradle"
 
 dependencies {
@@ -23,3 +27,42 @@ tasks.withType(Test).configureEach() {
     systemProperty 'zipkinExporterJar', project(':auto-exporters:opentelemetry-auto-exporters-zipkin').tasks.shadowJar.archivePath
   }
 }
+
+configurations {
+  shadowInclude
+}
+
+dependencies {
+  shadowInclude project(path: ':auto-exporters:opentelemetry-auto-exporters-logging', configuration: 'shadow')
+  shadowInclude project(path: ':auto-exporters:opentelemetry-auto-exporters-otlp', configuration: 'shadow')
+  shadowInclude project(path: ':auto-exporters:opentelemetry-auto-exporters-jaeger', configuration: 'shadow')
+  shadowInclude project(path: ':auto-exporters:opentelemetry-auto-exporters-zipkin', configuration: 'shadow')
+}
+
+
+shadowJar {
+  configurations = [project.configurations.shadowInclude]
+
+  archiveClassifier = ''
+
+  mergeServiceFiles()
+
+  exclude '**/module-info.class'
+
+  // Prevents conflict with other SLF4J instances. Important for premain.
+  relocate 'org.slf4j', 'io.opentelemetry.auto.slf4j'
+  // rewrite dependencies calling Logger.getLogger
+  relocate 'java.util.logging.Logger', 'io.opentelemetry.auto.bootstrap.PatchLogger'
+
+  // relocate OpenTelemetry API usage
+  relocate "io.opentelemetry.OpenTelemetry", "io.opentelemetry.auto.shaded.io.opentelemetry.OpenTelemetry"
+  relocate "io.opentelemetry.common", "io.opentelemetry.auto.shaded.io.opentelemetry.common"
+  relocate "io.opentelemetry.context", "io.opentelemetry.auto.shaded.io.opentelemetry.context"
+  relocate "io.opentelemetry.correlationcontext", "io.opentelemetry.auto.shaded.io.opentelemetry.correlationcontext"
+  relocate "io.opentelemetry.internal", "io.opentelemetry.auto.shaded.io.opentelemetry.internal"
+  relocate "io.opentelemetry.metrics", "io.opentelemetry.auto.shaded.io.opentelemetry.metrics"
+  relocate "io.opentelemetry.trace", "io.opentelemetry.auto.shaded.io.opentelemetry.trace"
+
+  // relocate OpenTelemetry API dependency usage
+  relocate "io.grpc", "io.opentelemetry.auto.shaded.io.grpc"
+}

auto-exporters/otlp/src/main/java/io/opentelemetry/auto/exporters/otlp/OtlpMetricExporterFactory.java

@@ -26,7 +26,7 @@ public class OtlpMetricExporterFactory implements MetricExporterFactory {
 
   @Override
   public MetricExporter fromConfig(final Config config) {
-    final String otlpEndpoint = config.getString(OTLP_ENDPOINT, "");
+    final String otlpEndpoint = config.getString(OTLP_ENDPOINT, "localhost:55680");
     if (otlpEndpoint.isEmpty()) {
       throw new IllegalStateException("ota.exporter.otlp.endpoint is required");
     }

auto-exporters/otlp/src/main/java/io/opentelemetry/auto/exporters/otlp/OtlpSpanExporterFactory.java

@@ -26,7 +26,7 @@ public class OtlpSpanExporterFactory implements SpanExporterFactory {
 
   @Override
   public SpanExporter fromConfig(final Config config) {
-    final String otlpEndpoint = config.getString(OTLP_ENDPOINT, "");
+    final String otlpEndpoint = config.getString(OTLP_ENDPOINT, "localhost:55680");
     if (otlpEndpoint.isEmpty()) {
       throw new IllegalStateException("ota.exporter.otlp.endpoint is required");
     }

auto-exporters/zipkin/zipkin.gradle

@@ -3,6 +3,7 @@ plugins {
 }
 
 apply from: "${rootDir}/gradle/java.gradle"
+apply from: "${rootDir}/gradle/publish.gradle"
 
 dependencies {
   compile(deps.opentelemetryZipkin) {

java-agent/java-agent.gradle

@@ -1,3 +1,5 @@
+import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar
+
 plugins {
   id "com.github.johnrengelman.shadow"
 }
@@ -12,17 +14,6 @@ configurations {
   shadowInclude
 }
 
-processResources {
-  from(zipTree(project(':instrumentation').tasks.shadowJar.archiveFile)) {
-    into 'auto-tooling-and-instrumentation.isolated'
-    rename '(^.*)\\.class$', '$1.classdata'
-    // Rename LICENSE file since it clashes with license dir on non-case sensitive FSs (i.e. Mac)
-    rename '^LICENSE$', 'LICENSE.renamed'
-  }
-
-  dependsOn project(':instrumentation').tasks.shadowJar
-}
-
 jar {
   manifest {
     attributes(
@@ -35,13 +26,50 @@ jar {
   }
 }
 
+CopySpec isolateSpec(Collection<Task> sourceTasks) {
+  return copySpec {
+    from(sourceTasks.collect { zipTree(it.archiveFile) }) {
+      into 'auto-tooling-and-instrumentation.isolated'
+      rename '(^.*)\\.class$', '$1.classdata'
+      // Rename LICENSE file since it clashes with license dir on non-case sensitive FSs (i.e. Mac)
+      rename '^LICENSE$', 'LICENSE.renamed'
+    }
+  }
+}
+
 shadowJar {
-  configurations = [project.configurations.shadowInclude]
+  archiveClassifier = 'no-exporters'
 
+  def sourceTasks = [project(':instrumentation').tasks.shadowJar]
+  dependsOn sourceTasks
+  with isolateSpec(sourceTasks)
+}
+
+task embedExporters(type: ShadowJar) {
   archiveClassifier = ''
+  from sourceSets.main.output
 
+  def sourceTasks = [project(':instrumentation').tasks.shadowJar, project(':auto-exporters').tasks.shadowJar]
+  dependsOn sourceTasks
+  with isolateSpec(sourceTasks)
+}
+assemble.dependsOn embedExporters
+
+publishing {
+  publications {
+    maven(MavenPublication) {
+      artifact embedExporters
+    }
+  }
+}
+tasks.withType(ShadowJar).configureEach {
+  configurations = [project.configurations.shadowInclude]
   mergeServiceFiles()
 
+  manifest {
+    inheritFrom project.tasks.jar.manifest
+  }
+
   exclude '**/module-info.class'
 
   dependencies {

smoke-tests/smoke-tests.gradle

@@ -10,12 +10,12 @@ dependencies {
 
 subprojects { subProject ->
   subProject.tasks.withType(Test).configureEach {
-    dependsOn ':opentelemetry-auto:shadowJar'
+    dependsOn = [':opentelemetry-auto:embedExporters', ':auto-exporters:opentelemetry-auto-exporters-logging:shadowJar']
 
     doFirst {
       // Tests depend on this to know where to run things and what agent jar to use
       jvmArgs "-Dio.opentelemetry.smoketest.builddir=${buildDir}"
-      jvmArgs "-Dio.opentelemetry.smoketest.agent.shadowJar.path=${project(':opentelemetry-auto').tasks.shadowJar.archivePath}"
+      jvmArgs "-Dio.opentelemetry.smoketest.agent.shadowJar.path=${project(':opentelemetry-auto').tasks.embedExporters.archivePath}"
       jvmArgs "-Dota.exporter.jar=${project(':auto-exporters:opentelemetry-auto-exporters-logging').tasks.shadowJar.archivePath}"
     }
   }

smoke-tests/springboot/src/test/groovy/io/opentelemetry/smoketest/SpringBootSmokeWithEmbeddedExporterTest.groovy

@@ -0,0 +1,64 @@
+/*
+ * Copyright The OpenTelemetry Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.opentelemetry.smoketest
+
+import okhttp3.Request
+
+/**
+ * This is almost an exact copy of {@link SpringBootSmokeTest}.
+ * The only difference is that this test does not use external exporter jar.
+ * It thus verifies that agent has embedded exporter and can use it.
+ */
+class SpringBootSmokeWithEmbeddedExporterTest extends AbstractServerSmokeTest {
+
+  static final HANDLER_SPAN = "LOGGED_SPAN WebController.greeting"
+  static final SERVLET_SPAN = "LOGGED_SPAN /greeting"
+
+  @Override
+  ProcessBuilder createProcessBuilder() {
+    String springBootShadowJar = System.getProperty("io.opentelemetry.smoketest.springboot.shadowJar.path")
+
+    List<String> command = new ArrayList<>()
+    command.add(javaPath())
+    command.addAll(defaultJavaProperties)
+    command.addAll((String[]) ["-Dota.exporter=logging", "-Dota.exporter.logging.prefix=LOGGED_SPAN", "-jar", springBootShadowJar, "--server.port=${httpPort}"])
+    ProcessBuilder processBuilder = new ProcessBuilder(command)
+    processBuilder.directory(new File(buildDirectory))
+  }
+
+  def "can monitor default home page"() {
+    setup:
+    def spanCounter = new SpanCounter(logfile, [
+      (HANDLER_SPAN): 1,
+      (SERVLET_SPAN): 1,
+    ], 10000)
+    String url = "http://localhost:${httpPort}/greeting"
+    def request = new Request.Builder().url(url).get().build()
+
+    when:
+    def response = client.newCall(request).execute()
+    def spans = spanCounter.countSpans()
+
+    then:
+    def responseBodyStr = response.body().string()
+    responseBodyStr != null
+    responseBodyStr.contains("Sup Dawg")
+    response.body().contentType().toString().contains("text/plain")
+    response.code() == 200
+    spans[HANDLER_SPAN] == 1
+    spans[SERVLET_SPAN] == 1
+  }
+}

smoke-tests/src/main/groovy/io/opentelemetry/smoketest/AbstractSmokeTest.groovy

@@ -21,10 +21,6 @@ import spock.lang.Specification
 
 abstract class AbstractSmokeTest extends Specification {
 
-  public static final API_KEY = "some-api-key"
-  public static final PROFILING_START_DELAY_SECONDS = 1
-  public static final int PROFILING_RECORDING_UPLOAD_PERIOD_SECONDS = 5
-
   @Shared
   protected String workingDirectory = System.getProperty("user.dir")
   @Shared
@@ -68,7 +64,6 @@ abstract class AbstractSmokeTest extends Specification {
     ProcessBuilder processBuilder = createProcessBuilder()
 
     processBuilder.environment().put("JAVA_HOME", System.getProperty("java.home"))
-    processBuilder.environment().put("DD_API_KEY", API_KEY)
 
     // Setting configuration variables of batch span processor through env vars
     // This config is to immediately flush a batch of 1 span with delay of 10ms