Merge pull request #620 from DataDog/ark/datadog-api-outline

datadog api outline

dd-trace-ext/README.md

@@ -0,0 +1,4 @@
+# Convenience Utils for Datadog Tracer Internal API
+- Trace Scopes
+- Global Tracer
+- Async controls

dd-trace-ext/dd-trace-ext.gradle

@@ -0,0 +1,6 @@
+description = 'dd-trace-ext'
+apply from: "${rootDir}/gradle/java.gradle"
+
+dependencies {
+  compile project(':dd-trace')
+}

dd-trace-ext/src/main/java/datadog/trace/tracer/ext/Examples.java

@@ -0,0 +1,76 @@
+package datadog.trace.tracer.ext;
+
+import datadog.trace.tracer.Span;
+import datadog.trace.tracer.Trace;
+import datadog.trace.tracer.Tracer;
+
+// Keeping in PR for potential discussions. Will eventually remove.
+// TODO: remove
+class Examples {
+  private Examples() {}
+
+  public static void test() {
+    final Throwable someThrowable = null;
+    // registration
+    TracerContext.registerGlobalContext(new TracerContext(new Tracer()), false);
+
+    // scope
+    final TracerContext ctx = TracerContext.getGlobalContext();
+    // without try-with-resources
+    {
+      Span rootSpan = ctx.getTracer().buildTrace(null);
+      final Scope scope = ctx.pushScope(rootSpan);
+      rootSpan.setError(true);
+      rootSpan.attachThrowable(someThrowable);
+      scope.close();
+      rootSpan.finish();
+    }
+
+    /*
+    // with try-with-resources finishOnClose=true
+    {
+      Span rootSpan = ctx.getTracer().buildTrace(null);
+      try (Scope scope = ctx.pushScope(rootSpan)) {
+        try {
+          // the body
+        } catch (Throwable t) {
+          rootSpan.setError(true);
+          rootSpan.attachThrowable(t);
+          throw t;
+        }
+      }
+    }
+    */
+
+    // with try-with-resources finishOnClose=false
+    {
+      Span rootSpan = ctx.getTracer().buildTrace(null);
+      try (Scope scope = ctx.pushScope(rootSpan)) {
+        // the body
+      } catch (Throwable t) {
+        rootSpan.setError(true);
+        rootSpan.attachThrowable(t);
+        throw t;
+      } finally {
+        rootSpan.finish();
+      }
+    }
+
+    // continuations
+    {
+      Span rootSpan = ctx.getTracer().buildTrace(null);
+      final Trace.Continuation cont = rootSpan.getTrace().createContinuation(rootSpan);
+      { // on another thread
+        final Span parent = cont.span();
+        try {
+          // body
+        } finally {
+          cont.close();
+        }
+      }
+    }
+
+    // create a span as a child of the currently active span
+    Span childSpan = ctx.peekScope().span().getTrace().createSpan(ctx.peekScope().span());
+  }
+}

dd-trace-ext/src/main/java/datadog/trace/tracer/ext/Scope.java

@@ -0,0 +1,26 @@
+package datadog.trace.tracer.ext;
+
+import datadog.trace.tracer.Span;
+import datadog.trace.tracer.Trace;
+
+/**
+ * A scope holds a single span or trace continuation and may optionally finish its span or
+ * continuation.
+ *
+ * <p>To create a scope, see {@link TracerContext#pushScope(Span)} and {@link
+ * TracerContext#pushScope(Trace.Continuation)}.
+ *
+ * <p>All created scopes must be closed with {@link Scope#close()}
+ */
+public interface Scope extends AutoCloseable {
+  /** Get the span held by this scope. */
+  Span span();
+
+  /**
+   * Close this scope. This method must be invoked on all created scopes.
+   *
+   * <p>Attempting to close a scope which is not on the top of its TracerContext's scope-stack is an
+   * error. See {@link TracerContext#peekScope()}.
+   */
+  void close();
+}

dd-trace-ext/src/main/java/datadog/trace/tracer/ext/TracerContext.java

@@ -0,0 +1,86 @@
+package datadog.trace.tracer.ext;
+
+import datadog.trace.tracer.Span;
+import datadog.trace.tracer.Trace;
+import datadog.trace.tracer.Tracer;
+
+/**
+ * Provides a place to store a shared global tracer with convenient span-building helper methods.
+ *
+ * <p>Also maintains a stack of active scopes (or scope-stack). The span of the scope on the top of
+ * the scope stack is used as the parent for newly created spans.
+ *
+ * <p>This class is thread safe.
+ */
+public final class TracerContext {
+  // global TracerContext
+  /** Get the global TracerContext */
+  public static TracerContext getGlobalContext() {
+    return null;
+  }
+
+  /**
+   * Register the global TracerContext.
+   *
+   * @param context The context to register.
+   * @param replaceExisting If true, the existing global TracerContext will be replaced
+   * @return The old global TracerContext, or null if no previous context ws registered
+   */
+  public static TracerContext registerGlobalContext(
+      TracerContext context, boolean replaceExisting) {
+    return null;
+  }
+
+  /** @return True if a global TracerContext has been registered */
+  public static boolean isRegistered() {
+    return false;
+  }
+
+  private final Tracer tracer;
+
+  public TracerContext(Tracer tracer) {
+    this.tracer = tracer;
+  }
+
+  /** @return The tracer associated with this TracerContext */
+  public Tracer getTracer() {
+    return tracer;
+  }
+
+  // TODO: convenience APIs like buildSpan, etc.
+
+  /**
+   * Push a new scope to the top of this scope-stack. The scope's span will be the given span.
+   *
+   * @param span
+   * @return
+   */
+  public Scope pushScope(Span span) {
+    return null;
+  }
+
+  /**
+   * Push a new scope to the top of this scope-stack. The scope's span will be the continuation's
+   * span.
+   *
+   * @param continuation
+   * @return
+   */
+  public Scope pushScope(Trace.Continuation continuation) {
+    return null;
+  }
+
+  /**
+   * Pop the given scope off the top of the scope stack.
+   *
+   * <p>If the given scope is not the topmost scope on the stack an error will be thrown.
+   *
+   * @param scope the topmost scope in the scope stack.
+   */
+  public void popScope(Scope scope) {}
+
+  /** @return The scope on the top of this scope-stack or null if there is no active scope. */
+  public Scope peekScope() {
+    return null;
+  }
+}

dd-trace/README.md

@@ -0,0 +1,4 @@
+# Datadog Tracer Internal API
+Contains the core elements needed to create and report APM traces to Datadog.
+
+It's recommended to use `:dd-trace-ext` in addition to this api.

dd-trace/dd-trace.gradle

@@ -0,0 +1,15 @@
+description = 'dd-trace'
+apply from: "${rootDir}/gradle/java.gradle"
+
+dependencies {
+  annotationProcessor deps.autoservice
+  implementation deps.autoservice
+
+  compile project(':dd-trace-api')
+
+  compile deps.jackson
+  compile deps.slf4j
+  // any higher versions seems to break ES tests with this exception:
+  // java.lang.NoSuchMethodError: com.fasterxml.jackson.dataformat.smile.SmileGenerator.getOutputContext()
+  compile group: 'org.msgpack', name: 'jackson-dataformat-msgpack', version: '0.8.14'
+}

dd-trace/src/main/java/datadog/trace/tracer/Clock.java

@@ -0,0 +1,56 @@
+package datadog.trace.tracer;
+
+import java.util.concurrent.TimeUnit;
+
+/**
+ * A simple wrapper for system clock that aims to provide the current time
+ *
+ * <p>
+ *
+ * <p>
+ *
+ * <p>
+ *
+ * <p>The JDK provides two clocks:
+ * <li>one in nanoseconds, for precision, but it can only use to measure durations
+ * <li>one in milliseconds, for accuracy, useful to provide epoch time
+ *
+ *     <p>
+ *
+ *     <p>At this time, we are using a millis precision (converted to micros) in order to guarantee
+ *     consistency between the span start times and the durations
+ */
+public class Clock {
+
+  /**
+   * Get the current nanos ticks (i.e. System.nanonTime()), this method can't be use for date
+   * accuracy (only duration calculations)
+   *
+   * @return The current nanos ticks
+   */
+  public static long nanoTime() {
+    return System.nanoTime();
+  }
+
+  /**
+   * Get the current epoch time in micros.
+   *
+   * <p>Note: The actual precision is the millis.
+   *
+   * @return the current epoch time in micros
+   */
+  public static long epochTimeMicro() {
+    return TimeUnit.MILLISECONDS.toMicros(System.currentTimeMillis());
+  }
+
+  /**
+   * Get the current epoch time in nanos.
+   *
+   * <p>Note: The actual precision is the millis. This will overflow ~290 years after epoch
+   *
+   * @return the current epoch time in nanos
+   */
+  public static long epochTimeNano() {
+    return TimeUnit.MILLISECONDS.toNanos(System.currentTimeMillis());
+  }
+}

dd-trace/src/main/java/datadog/trace/tracer/Span.java

@@ -0,0 +1,117 @@
+package datadog.trace.tracer;
+
+/**
+ * A single measurement of time with arbitrary key-value attributes.
+ *
+ * <p>All spans are thread safe.
+ *
+ * <p>To create a Span, see {@link Tracer#buildTrace()}
+ */
+public interface Span {
+
+  /** @return The trace this span is associated with. */
+  Trace getTrace();
+
+  /** Stop the span's timer. Has no effect if the span is already finished. */
+  void finish();
+
+  /**
+   * Stop the span's timer. Has no effect if the span is already finished.
+   *
+   * <p>It's undefined behavior to specify a finish timestamp which occurred before this span's
+   * start timestamp.
+   *
+   * @param finishTimestampNanoseconds Epoch time in nanoseconds.
+   */
+  void finish(long finishTimestampNanoseconds);
+
+  /** Returns true if a finish method has been invoked on this span. */
+  boolean isFinished();
+
+  /**
+   * Get the span context for this span.
+   *
+   * @return the span context.
+   */
+  SpanContext getContext();
+
+  /**
+   * Get a meta value on a span.
+   *
+   * @param key The meta key
+   * @return The value currently associated with the given key. Null if no associated. TODO: can
+   *     return null
+   */
+  Object getMeta(String key);
+
+  /**
+   * Set key-value metadata on the span.
+   *
+   * <p>TODO: Forbid setting null?
+   */
+  void setMeta(String key, String value);
+
+  /** {@link Span#setMeta(String, String)} for boolean values */
+  void setMeta(String key, boolean value);
+
+  /** {@link Span#setMeta(String, String)} for number values */
+  void setMeta(String key, Number value);
+
+  /** Get the span's name */
+  String getName();
+  /**
+   * Set the span's name.
+   *
+   * @param newName the new name for the span.
+   */
+  void setName(String newName);
+
+  String getResource();
+
+  void setResource(String newResource);
+
+  String getService();
+
+  void setService(String newService);
+
+  String getType();
+
+  void setType(String newType);
+
+  boolean isErrored();
+
+  /** Attach a throwable to this span. */
+  void attachThrowable(Throwable t);
+
+  /**
+   * Mark the span as having an error.
+   *
+   * @param isErrored true if the span has an error.
+   */
+  void setError(boolean isErrored);
+
+  // TODO: OpenTracing Span#log methods. Do we need something here to support them? Current DDSpan
+  // does not implement.
+
+  /**
+   * A Interceptor allows adding hooks to particular events between a span starting and finishing.
+   */
+  interface Interceptor {
+    /**
+     * Called after a span is started.
+     *
+     * @param span the started span
+     */
+    void spanStarted(Span span);
+
+    /** Called after a span's metadata is updated. */
+    void afterMetadataSet(Span span, Object key, Object value);
+
+    /**
+     * Called after a span is finished.
+     *
+     * @param span the started span
+     */
+    void spanFinished(Span span);
+  }
+}

dd-trace/src/main/java/datadog/trace/tracer/SpanContext.java

@@ -0,0 +1,39 @@
+package datadog.trace.tracer;
+
+/**
+ * All attributes of a {@link Span} which propagate for distributed tracing.
+ *
+ * <p>All Spans must have a SpanContext, but not all SpanContexts require a span.
+ *
+ * <p>All SpanContexts are thread safe.
+ */
+public interface SpanContext {
+  /**
+   * Get this context's span id.
+   *
+   * @return 64 bit unsigned integer in String format.
+   */
+  String getSpanId();
+
+  /**
+   * Get this context's parent span id.
+   *
+   * @return 64 bit unsigned integer in String format.
+   */
+  String getParentId();
+
+  /**
+   * Get this context's trace id.
+   *
+   * @return 64 bit unsigned integer in String format.
+   */
+  String getTraceId();
+
+  /**
+   * Get the sampling flag for this context.
+   *
+   * @return sampling flag for null if no sampling flags are set.
+   */
+  // TODO: should we add a @Nullable annotation to our project?
+  Integer getSamplingFlags();
+}

dd-trace/src/main/java/datadog/trace/tracer/SpanImpl.java

@@ -0,0 +1,138 @@
+package datadog.trace.tracer;
+
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicLong;
+
+public class SpanImpl implements Span {
+  private final Clock clock = null;
+  private final Trace trace = null;
+  // See See https://docs.datadoghq.com/api/?lang=python#tracing
+  // Required attributes to report to datadog.
+  private final SpanContext context = null;
+  /* Span name. May not exceed 100 characters. */
+  private final String name = "";
+  /* Span resource (e.g. http endpoint). May not exceed 5000 characters. */
+  private final String resource = "";
+  /* Span service. May not exceed 100 characters. */
+  private final String service = "";
+  /* The start time of the request in nanoseconds from the unix epoch. */
+  private final long startEpochNano = -1;
+  /* Duration of the span in nanoseconds */
+  private final AtomicLong durationNano = new AtomicLong(-1);
+  // optional attributes to report to datadog
+  /* The type of the span (web, db, etc). See DDSpanTypes. */
+  private final String type = null;
+  /* Marks the span as having an error. */
+  private final boolean isErrored = false;
+  /* Additional key-value pairs for a span. */
+  private final Map<String, Object> meta = null;
+
+  /**
+   * Create a span with a start time of the current timestamp.
+   *
+   * @param trace The trace to associate this span with
+   * @param parentContext identifies the parent of this span. May be null.
+   * @param clock The clock to use to measure the span's duration.
+   * @param interceptors interceptors to run on the span
+   */
+  SpanImpl(
+      final TraceImpl trace,
+      final SpanContext parentContext,
+      final Clock clock,
+      List<Interceptor> interceptors) {}
+
+  /**
+   * Create a span with the a specific start timestamp.
+   *
+   * @param trace The trace to associate this span with
+   * @param parentContext identifies the parent of this span. May be null.
+   * @param clock The clock to use to measure the span's duration.
+   * @param interceptors interceptors to run on the span
+   * @param startTimestampNanoseconds Epoch time in nanoseconds when this span started.
+   */
+  SpanImpl(
+      final TraceImpl trace,
+      final SpanContext parentContext,
+      final Clock clock,
+      List<Interceptor> interceptors,
+      final long startTimestampNanoseconds) {}
+
+  @Override
+  public Trace getTrace() {
+    return null;
+  }
+
+  @Override
+  public void finish() {}
+
+  @Override
+  public void finish(long finishTimestampNanoseconds) {}
+
+  @Override
+  public boolean isFinished() {
+    return false;
+  }
+
+  @Override
+  public void attachThrowable(Throwable t) {}
+
+  @Override
+  public void setError(boolean isErrored) {}
+
+  @Override
+  public SpanContext getContext() {
+    return null;
+  }
+
+  @Override
+  public Object getMeta(String key) {
+    return null;
+  }
+
+  @Override
+  public void setMeta(String key, String value) {}
+
+  @Override
+  public void setMeta(String key, boolean value) {}
+
+  @Override
+  public void setMeta(String key, Number value) {}
+
+  @Override
+  public String getName() {
+    return null;
+  }
+
+  @Override
+  public void setName(String newName) {}
+
+  @Override
+  public String getResource() {
+    return null;
+  }
+
+  @Override
+  public void setResource(String newResource) {}
+
+  @Override
+  public String getService() {
+    return null;
+  }
+
+  @Override
+  public void setService(String newService) {}
+
+  @Override
+  public String getType() {
+    return null;
+  }
+
+  @Override
+  public void setType(String newType) {}
+
+  @Override
+  public boolean isErrored() {
+    return false;
+  }
+}

dd-trace/src/main/java/datadog/trace/tracer/Trace.java

@@ -0,0 +1,59 @@
+package datadog.trace.tracer;
+
+/**
+ * A tree of {@link Span}s with a single root node plus logic to determine when to report said tree
+ * to the backend.
+ *
+ * <p>A trace will be written when all of its spans are finished and all trace continuations are
+ * closed.
+ *
+ * <p>To create a Trace, see {@link Tracer#buildTrace()}
+ */
+public interface Trace {
+  /** Get the tracer which created this trace. */
+  Tracer getTracer();
+
+  /** Get the root span for this trace. This will never be null. */
+  Span getRootSpan();
+
+  /**
+   * Create a new span in this trace as a child of the given parentSpan.
+   *
+   * @param parentSpan the parent to use. Must be a span in this trace.
+   * @return the new span. It is the caller's responsibility to ensure {@link Span#finish()} is
+   *     eventually invoked on this span.
+   */
+  Span createSpan(Span parentSpan);
+
+  /**
+   * Create a new continuation for this trace
+   *
+   * @param parentSpan the parent to use. Must be a span in this trace.
+   * @return the new continuation. It is the caller's responsibility to ensure {@link
+   *     Continuation#close()} is eventually invoked on this continuation.
+   */
+  Continuation createContinuation(Span parentSpan);
+
+  interface Interceptor {
+    /**
+     * Invoked when a trace is eligible for writing but hasn't been handed off to its writer yet.
+     *
+     * @param trace The intercepted trace.
+     */
+    void beforeTraceWritten(Trace trace);
+  }
+
+  /** A way to prevent a trace from reporting without creating a span. */
+  interface Continuation {
+    /**
+     * Close the continuation. Continuation's trace will not block reporting on account of this
+     * continuation.
+     *
+     * <p>Has no effect after the first invocation.
+     */
+    void close();
+
+    // TODO: doc
+    Span span();
+  }
+}

dd-trace/src/main/java/datadog/trace/tracer/TraceImpl.java

@@ -0,0 +1,54 @@
+package datadog.trace.tracer;
+
+import datadog.trace.tracer.writer.Writer;
+import java.lang.ref.WeakReference;
+import java.util.List;
+import java.util.Set;
+
+public class TraceImpl implements Trace {
+  private final Writer writer = null;
+
+  // TODO: Document approach to weak-referencdes and cleanup. If a span has to be closed by our GC
+  // logic the trace should increment the writer's count but not report (invalid api usage produces
+  // suspect data).
+  private final Set<WeakReference<Span>> inFlightSpans = null;
+  private final Set<WeakReference<Trace.Continuation>> inFlightContinuations = null;
+
+  /** Strong refs to spans which are closed */
+  private final List<Span> finishedSpans = null;
+
+  private final Span rootSpan = null;
+
+  /**
+   * Create a new Trace.
+   *
+   * @param tracer the Tracer to apply settings from.
+   */
+  TraceImpl(
+      Tracer tracer,
+      SpanContext rootSpanParentContext,
+      final long rootSpanStartTimestampNanoseconds) {}
+
+  @Override
+  public Tracer getTracer() {
+    return null;
+  }
+
+  @Override
+  public Span getRootSpan() {
+    return null;
+  }
+
+  @Override
+  public Span createSpan(Span parentSpan) {
+    return null;
+  }
+
+  @Override
+  public Continuation createContinuation(Span parentSpan) {
+    return null;
+  }
+
+  // TODO methods to inform the trace that continuations and spans finished/closed. Also be able to
+  // inform trace when a span finishes due to GC.
+}

dd-trace/src/main/java/datadog/trace/tracer/Tracer.java

@@ -0,0 +1,47 @@
+package datadog.trace.tracer;
+
+import datadog.trace.api.Config;
+import datadog.trace.tracer.sampling.Sampler;
+import datadog.trace.tracer.writer.Writer;
+
+/** A Tracer creates {@link Trace}s and holds common settings across traces. */
+public class Tracer {
+  /** Default service name if none provided on the trace or span */
+  private final String defaultServiceName = null;
+  /** Writer is an charge of reporting traces and spans to the desired endpoint */
+  private final Writer writer = null;
+  /** Sampler defines the sampling policy in order to reduce the number of traces for instance */
+  private final Sampler sampler = null;
+  /** Settings for this tracer. */
+  private final Config config = null;
+  /** The clock to use for tracing. */
+  private final Clock clock = null;
+
+  /**
+   * Construct a new trace using this tracer's settings and return the root span.
+   *
+   * @return The root span of the new trace.
+   */
+  public Span buildTrace(final SpanContext parentContext) {
+    return null;
+  }
+
+  /**
+   * Construct a new trace using this tracer's settings and return the root span.
+   *
+   * @param rootSpanStartTimestampNanoseconds Epoch time in nanoseconds when the root span started.
+   * @return The root span of the new trace.
+   */
+  public Span buildTrace(
+      final SpanContext parentContext, final long rootSpanStartTimestampNanoseconds) {
+    return null;
+  }
+
+  // TODO: doc inject and extract
+  // TODO: inject and extract helpers on span context?
+  public <T> void inject(final SpanContext spanContext, final Object format, final T carrier) {}
+
+  public <T> SpanContext extract(final Object format, final T carrier) {
+    return null;
+  }
+}

dd-trace/src/main/java/datadog/trace/tracer/sampling/Sampler.java

@@ -0,0 +1,19 @@
+package datadog.trace.tracer.sampling;
+
+import datadog.trace.tracer.Trace;
+
+/**
+ * Keeps or discards traces.
+ *
+ * <p>Note that in most cases the sampler will keep all traces. Most of the sampling logic is done
+ * downstream by the trace-agent or dd-backend.
+ */
+public interface Sampler {
+  /**
+   * Run tracer sampling logic on the trace.
+   *
+   * @param trace
+   * @return true if the trace should be kept/written/reported.
+   */
+  boolean sample(Trace trace);
+}

dd-trace/src/main/java/datadog/trace/tracer/writer/Writer.java

@@ -0,0 +1,27 @@
+package datadog.trace.tracer.writer;
+
+import datadog.trace.tracer.Trace;
+
+/** A writer sends traces to some place. */
+public interface Writer {
+  /**
+   * Write a trace represented by the entire list of all the finished spans
+   *
+   * @param trace the trace to write
+   */
+  void write(Trace trace);
+
+  /**
+   * Inform the writer that a trace occurred but will not be written. Used by tracer-side sampling.
+   */
+  void incrementTraceCount();
+
+  /** Start the writer */
+  void start();
+
+  /**
+   * Indicates to the writer that no future writing will come and it should terminates all
+   * connections and tasks
+   */
+  void close();
+}

settings.gradle

@@ -1,13 +1,22 @@
 rootProject.name = 'dd-trace-java'
 
-include ':dd-trace-ot'
+// core tracing projects
-include ':dd-java-agent'
+include ':dd-trace-api'
 include ':dd-java-agent:agent-bootstrap'
+include ':dd-trace'
+include ':dd-trace-ext'
+
+// implements for third-party tracing libraries
+include ':dd-trace-ot'
+
+// agent projects
+include ':dd-java-agent'
 include ':dd-java-agent:agent-tooling'
 include ':dd-java-agent:agent-jmxfetch'
+
+// misc
 include ':dd-java-agent:testing'
 include ':dd-java-agent-ittests'
-include ':dd-trace-api'
 
 // instrumentation:
 include ':dd-java-agent:instrumentation:akka-http-10.0'