Merge pull request #70 from DataDog/gpolert/mongo-norm

Normalize mongo queries

dd-java-agent/src/main/java/com/datadoghq/trace/agent/integration/MongoHelper.java

@@ -1,13 +1,27 @@
 package com.datadoghq.trace.agent.integration;
 
+import com.datadoghq.trace.DDTags;
 import com.mongodb.MongoClientOptions;
+import com.mongodb.event.CommandStartedEvent;
+import io.opentracing.Span;
 import io.opentracing.contrib.mongo.TracingCommandListener;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import lombok.extern.slf4j.Slf4j;
+import org.bson.BsonDocument;
+import org.bson.BsonString;
+import org.bson.BsonValue;
 import org.jboss.byteman.rule.Rule;
 
 /** Patch the Mongo builder before constructing the final client */
+@Slf4j
 public class MongoHelper extends DDAgentTracingHelper<MongoClientOptions.Builder> {
 
-  public MongoHelper(Rule rule) {
+  private static final List<String> WHILDCARD_FIELDS = Arrays.asList("ordered", "insert");
+  private static final BsonValue HIDDEN_CAR = new BsonString("?");
+
+  public MongoHelper(final Rule rule) {
     super(rule);
   }
 
@@ -20,14 +34,41 @@ public class MongoHelper extends DDAgentTracingHelper<MongoClientOptions.Builder
    *     client construction
    * @throws Exception
    */
-  protected MongoClientOptions.Builder doPatch(MongoClientOptions.Builder builder)
+  @Override
+  protected MongoClientOptions.Builder doPatch(final MongoClientOptions.Builder builder)
       throws Exception {
 
-    TracingCommandListener listener = new TracingCommandListener(tracer);
+    final TracingCommandListener listener = new TracingCommandListener(tracer);
     builder.addCommandListener(listener);
 
     setState(builder, 1);
 
     return builder;
   }
+
+  public void decorate(final Span span, final CommandStartedEvent event) {
+    try {
+      final BsonDocument normalized = new BsonDocument();
+      norm(event.getCommand(), normalized);
+      span.setTag(DDTags.RESOURCE_NAME, normalized.toString());
+      span.setOperationName("mongo.cmd");
+
+    } catch (final Throwable e) {
+      log.warn("Couldn't decorate the mongo query: " + e.getMessage(), e);
+    }
+  }
+
+  private void norm(final BsonDocument origin, final BsonDocument normalized) {
+    for (final Map.Entry<String, BsonValue> entry : origin.entrySet()) {
+      if (WHILDCARD_FIELDS.contains(entry.getKey())) {
+        normalized.put(entry.getKey(), entry.getValue());
+      } else if (entry.getValue().isDocument()) {
+        final BsonDocument child = new BsonDocument();
+        normalized.put(entry.getKey(), child);
+        norm(entry.getValue().asDocument(), child);
+      } else {
+        normalized.put(entry.getKey(), HIDDEN_CAR);
+      }
+    }
+  }
 }

dd-java-agent/src/main/resources/otarules.btm

@@ -20,6 +20,17 @@ DO
   patch($this);
 ENDRULE
 
+RULE mongo-decorator
+CLASS io.opentracing.contrib.mongo.TracingCommandListener
+METHOD decorate
+HELPER com.datadoghq.trace.agent.integration.MongoHelper
+AT EXIT
+IF TRUE
+DO
+  decorate($1, $2);
+ENDRULE
+
+
 
 # Instrument AWS SDK client
 # ==========================

dd-trace-examples/rest-spark/rest-spark.gradle

@@ -16,4 +16,8 @@ dependencies {
   compile 'io.opentracing:opentracing-api:0.30.0'
   compile 'io.opentracing:opentracing-util:0.30.0'
   compile 'io.opentracing.contrib:opentracing-mongo-driver:0.0.2'
+  compile group: 'ch.qos.logback', name: 'logback-classic', version: '1.2.3'
+  compile group: 'org.slf4j', name: 'slf4j-api', version: '1.7.25'
+
+
 }

dd-trace/src/main/java/com/datadoghq/trace/integration/DBStatementAsResourceName.java

@@ -10,4 +10,35 @@ public class DBStatementAsResourceName extends DDSpanContextDecorator {
     this.setMatchingTag(Tags.DB_STATEMENT.getKey());
     this.setSetTag(DDTags.RESOURCE_NAME);
   }
+
+  //{ "insert" : "calls", "ordered" : true, "documents" : [{ "_id" : { "$oid" : "5979bbb0ed6fed5749cc9e7c" }, "name" : "MongoDB", "type" : "database", "identifier" : "10", "versions" : ["v3.2", "v3.0", "v2.6"], "info" : { "x" : 203, "y" : 102 } }] }
+  private void normalizeFilter(final Object f) {}
 }
+
+  /*
+  def normalize_filter(f=None):
+      if f is None:
+          return {}
+      elif isinstance(f, list):
+          # normalize lists of filters
+          # e.g. {$or: [ { age: { $lt: 30 } }, { type: 1 } ]}
+          return [normalize_filter(s) for s in f]
+      elif isinstance(f, dict):
+          # normalize dicts of filters
+          #   {$or: [ { age: { $lt: 30 } }, { type: 1 } ]})
+          out = {}
+          for k, v in iteritems(f):
+              if k == "$in" or k == "$nin":
+                  # special case $in queries so we don't loop over lists.
+                  out[k] = "?"
+              elif isinstance(v, list) or isinstance(v, dict):
+                  # RECURSION ALERT: needs to move to the agent
+                  out[k] = normalize_filter(v)
+              else:
+                  # NOTE: this shouldn't happen, but let's have a safeguard.
+                  out[k] = '?'
+          return out
+      else:
+          # FIXME[matt] unexpected type. not sure this should ever happen, but at
+          # least it won't crash.
+          return {}*/

dd-trace/src/main/resources/dd-trace-decorators.yaml

@@ -8,5 +8,5 @@ decorators:
   - type: HTTPComponent
     matchingValue: java-aws-sdk
     setValue: aws-client
-  - type: DBStatementAsResourceName
+#  - type: DBStatementAsResourceName
   - type: ErrorFlag