From de166e5ecb994072ceb403e4aabc4a708fcedce8 Mon Sep 17 00:00:00 2001 From: Tyler Benson Date: Wed, 2 Jan 2019 10:47:28 -0800 Subject: [PATCH 1/4] First pass at embedding existing jmx integrations Using copy/paste from integrations-core. Will depend on a new jmxfetch release with https://github.com/DataDog/jmxfetch/pull/205 before this will work. --- .../trace/agent/jmxfetch/JMXFetch.java | 35 +- .../trace/agent/jmxfetch/metricconfigs.txt | 5 + .../jmxfetch/metricconfigs/activemq.yaml | 54 ++ .../jmxfetch/metricconfigs/cassandra.yaml | 427 +++++++++++++ .../agent/jmxfetch/metricconfigs/kafka.yaml | 566 ++++++++++++++++++ .../agent/jmxfetch/metricconfigs/solr.yaml | 200 +++++++ .../agent/jmxfetch/metricconfigs/tomcat.yaml | 65 ++ 7 files changed, 1351 insertions(+), 1 deletion(-) create mode 100644 dd-java-agent/agent-jmxfetch/src/main/resources/datadog/trace/agent/jmxfetch/metricconfigs.txt create mode 100644 dd-java-agent/agent-jmxfetch/src/main/resources/datadog/trace/agent/jmxfetch/metricconfigs/activemq.yaml create mode 100644 dd-java-agent/agent-jmxfetch/src/main/resources/datadog/trace/agent/jmxfetch/metricconfigs/cassandra.yaml create mode 100644 dd-java-agent/agent-jmxfetch/src/main/resources/datadog/trace/agent/jmxfetch/metricconfigs/kafka.yaml create mode 100644 dd-java-agent/agent-jmxfetch/src/main/resources/datadog/trace/agent/jmxfetch/metricconfigs/solr.yaml create mode 100644 dd-java-agent/agent-jmxfetch/src/main/resources/datadog/trace/agent/jmxfetch/metricconfigs/tomcat.yaml diff --git a/dd-java-agent/agent-jmxfetch/src/main/java/datadog/trace/agent/jmxfetch/JMXFetch.java b/dd-java-agent/agent-jmxfetch/src/main/java/datadog/trace/agent/jmxfetch/JMXFetch.java index 99770f6c75..83d28550f3 100644 --- a/dd-java-agent/agent-jmxfetch/src/main/java/datadog/trace/agent/jmxfetch/JMXFetch.java +++ b/dd-java-agent/agent-jmxfetch/src/main/java/datadog/trace/agent/jmxfetch/JMXFetch.java @@ -2,9 +2,16 @@ package datadog.trace.agent.jmxfetch; import com.google.common.collect.ImmutableList; import datadog.trace.api.Config; +import java.io.IOException; +import java.io.InputStream; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Map; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.io.IOUtils; import org.datadog.jmxfetch.App; import org.datadog.jmxfetch.AppConfig; @@ -27,6 +34,7 @@ public class JMXFetch { return; } + final List internalMetricsConfigs = getInternalMetricFiles(); final List metricsConfigs = config.getJmxFetchMetricsConfigs(); final Integer checkPeriod = config.getJmxFetchCheckPeriod(); final Integer refreshBeansPeriod = config.getJmxFetchRefreshBeansPeriod(); @@ -36,7 +44,8 @@ public class JMXFetch { final String logLevel = getLogLevel(); log.error( - "JMXFetch config: {} {} {} {} {} {} {}", + "JMXFetch config: {} {} {} {} {} {} {} {}", + internalMetricsConfigs, metricsConfigs, checkPeriod, refreshBeansPeriod, @@ -47,6 +56,7 @@ public class JMXFetch { final AppConfig appConfig = AppConfig.create( DEFAULT_CONFIGS, + internalMetricsConfigs, metricsConfigs, checkPeriod, refreshBeansPeriod, @@ -95,6 +105,29 @@ public class JMXFetch { return "statsd:" + host + ":" + config.getJmxFetchStatsdPort(); } + private static List getInternalMetricFiles() { + try { + final InputStream metricConfigsStream = + JMXFetch.class.getResourceAsStream("metricconfigs.txt"); + if (metricConfigsStream == null) { + log.debug("metricconfigs not found. returning empty set"); + return Collections.emptyList(); + } else { + final String configs = IOUtils.toString(metricConfigsStream, StandardCharsets.UTF_8); + final String[] split = configs.split("\n"); + final List result = new ArrayList<>(split.length); + for (final String config : split) { + final URL resource = JMXFetch.class.getResource("metricconfigs/" + config); + result.add(resource.getPath().split("\\.jar!/")[1]); + } + return result; + } + } catch (final IOException e) { + log.debug("error reading metricconfigs. returning empty set", e); + return Collections.emptyList(); + } + } + private static String getLogLocation() { return System.getProperty("org.slf4j.simpleLogger.logFile", "System.err"); } diff --git a/dd-java-agent/agent-jmxfetch/src/main/resources/datadog/trace/agent/jmxfetch/metricconfigs.txt b/dd-java-agent/agent-jmxfetch/src/main/resources/datadog/trace/agent/jmxfetch/metricconfigs.txt new file mode 100644 index 0000000000..521c7d2204 --- /dev/null +++ b/dd-java-agent/agent-jmxfetch/src/main/resources/datadog/trace/agent/jmxfetch/metricconfigs.txt @@ -0,0 +1,5 @@ +activemq.yaml +cassandra.yaml +kafka.yaml +solr.yaml +tomcat.yaml diff --git a/dd-java-agent/agent-jmxfetch/src/main/resources/datadog/trace/agent/jmxfetch/metricconfigs/activemq.yaml b/dd-java-agent/agent-jmxfetch/src/main/resources/datadog/trace/agent/jmxfetch/metricconfigs/activemq.yaml new file mode 100644 index 0000000000..d0735950ba --- /dev/null +++ b/dd-java-agent/agent-jmxfetch/src/main/resources/datadog/trace/agent/jmxfetch/metricconfigs/activemq.yaml @@ -0,0 +1,54 @@ +# Default metrics collected by this check. You should not have to modify this. +jmx_metrics: +- include: + destinationType: Queue + attribute: + AverageEnqueueTime: + alias: activemq.queue.avg_enqueue_time + metric_type: gauge + ConsumerCount: + alias: activemq.queue.consumer_count + metric_type: gauge + ProducerCount: + alias: activemq.queue.producer_count + metric_type: gauge + MaxEnqueueTime: + alias: activemq.queue.max_enqueue_time + metric_type: gauge + MinEnqueueTime: + alias: activemq.queue.min_enqueue_time + metric_type: gauge + MemoryPercentUsage: + alias: activemq.queue.memory_pct + metric_type: gauge + QueueSize: + alias: activemq.queue.size + metric_type: gauge + DequeueCount: + alias: activemq.queue.dequeue_count + metric_type: counter + DispatchCount: + alias: activemq.queue.dispatch_count + metric_type: counter + EnqueueCount: + alias: activemq.queue.enqueue_count + metric_type: counter + ExpiredCount: + alias: activemq.queue.expired_count + metric_type: counter + InFlightCount: + alias: activemq.queue.in_flight_count + metric_type: counter + +- include: + type: Broker + attribute: + StorePercentUsage: + alias: activemq.broker.store_pct + metric_type: gauge + TempPercentUsage: + alias: activemq.broker.temp_pct + metric_type: gauge + MemoryPercentUsage: + alias: activemq.broker.memory_pct + metric_type: gauge diff --git a/dd-java-agent/agent-jmxfetch/src/main/resources/datadog/trace/agent/jmxfetch/metricconfigs/cassandra.yaml b/dd-java-agent/agent-jmxfetch/src/main/resources/datadog/trace/agent/jmxfetch/metricconfigs/cassandra.yaml new file mode 100644 index 0000000000..a94c31374d --- /dev/null +++ b/dd-java-agent/agent-jmxfetch/src/main/resources/datadog/trace/agent/jmxfetch/metricconfigs/cassandra.yaml @@ -0,0 +1,427 @@ +# Default metrics collected by this check. You should not have to modify this. +jmx_metrics: + - include: + domain: org.apache.cassandra.metrics + type: ClientRequest + name: + - Latency + attribute: + - 75thPercentile + - 95thPercentile + - OneMinuteRate + - include: + domain: org.apache.cassandra.metrics + type: DroppedMessage + name: + - Dropped + attribute: + - OneMinuteRate + - include: + domain: org.apache.cassandra.metrics + type: ThreadPools + scope: + - MutationStage + - CounterMutationStage + - ReadStage + - ViewMutationStage + name: + - PendingTasks + - CurrentlyBlockedTasks + path: + - request + - include: + domain: org.apache.cassandra.metrics + type: ThreadPools + scope: + - MemtableFlushWriter + - HintsDispatcher + - MemtablePostFlush + - MigrationStage + - MiscStage + - SecondaryIndexManagement + name: + - PendingTasks + - CurrentlyBlockedTasks + path: + - internal + - include: + domain: org.apache.cassandra.metrics + type: Storage + name: + - Load + - Exceptions + - include: + domain: org.apache.cassandra.metrics + type: Table + bean_regex: + - .*keyspace=.* + name: + - ReadLatency + - WriteLatency + attribute: + - 75thPercentile + - 95thPercentile + - 99thPercentile + - OneMinuteRate + exclude: + keyspace: + - system + - system_auth + - system_distributed + - system_schema + - system_traces + - include: + domain: org.apache.cassandra.metrics + type: Table + bean_regex: + - .*keyspace=.* + name: + - RangeLatency + - CasPrepareLatency + - CasProposeLatency + - CasCommitLatency + - ViewLockAcquireTime + - ViewReadTime + attribute: + - 75thPercentile + - 95thPercentile + - OneMinuteRate + exclude: + keyspace: + - system + - system_auth + - system_distributed + - system_schema + - system_traces + - include: + domain: org.apache.cassandra.metrics + type: Table + bean_regex: + - .*keyspace=.* + name: + - SSTablesPerReadHistogram + - TombstoneScannedHistogram + - WaitingOnFreeMemtableSpace + attribute: + - 75thPercentile + - 95thPercentile + exclude: + keyspace: + - system + - system_auth + - system_distributed + - system_schema + - system_traces + - include: + domain: org.apache.cassandra.metrics + type: Table + bean_regex: + - .*keyspace=.* + name: + - ColUpdateTimeDeltaHistogram + attribute: + - Min + - 75thPercentile + - 95thPercentile + exclude: + keyspace: + - system + - system_auth + - system_distributed + - system_schema + - system_traces + - include: + domain: org.apache.cassandra.metrics + type: Table + bean_regex: + - .*keyspace=.* + name: + - BloomFilterFalseRatio + - CompressionRatio + - KeyCacheHitRate + - LiveSSTableCount + - MaxPartitionSize + - MeanPartitionSize + - MeanRowSize + - MaxRowSize + - PendingCompactions + - SnapshotsSize + attribute: + - Value + exclude: + keyspace: + - system + - system_auth + - system_distributed + - system_schema + - system_traces + - include: + domain: org.apache.cassandra.metrics + type: Table + bean_regex: + - .*keyspace=.* + name: + - CompactionBytesWritten + - BytesFlushed + - PendingFlushes + - LiveDiskSpaceUsed + - TotalDiskSpaceUsed + - RowCacheHitOutOfRange + - RowCacheHit + - RowCacheMiss + attribute: + - Count + exclude: + keyspace: + - system + - system_auth + - system_distributed + - system_schema + - system_traces + - include: + domain: org.apache.cassandra.metrics + type: Cache + scope: KeyCache + name: + - HitRate + attribute: + - Count + - include: + domain: org.apache.cassandra.metrics + type: CommitLog + name: + - PendingTasks + - TotalCommitLogSize + attribute: + - Value + - include: + domain: org.apache.cassandra.db + type: Tables + attribute: + DroppableTombstoneRatio: + alias: cassandra.db.droppable_tombstone_ratio + # Young Gen Collectors (Minor Collections) + - include: + domain: java.lang + type: GarbageCollector + name: Copy + attribute: + CollectionCount: + metric_type: counter + alias: jmx.gc.minor_collection_count + CollectionTime: + metric_type: counter + alias: jmx.gc.minor_collection_time + - include: + domain: java.lang + type: GarbageCollector + name: PS Scavenge + attribute: + CollectionCount: + metric_type: counter + alias: jmx.gc.minor_collection_count + CollectionTime: + metric_type: counter + alias: jmx.gc.minor_collection_time + - include: + domain: java.lang + type: GarbageCollector + name: ParNew + attribute: + CollectionCount: + metric_type: counter + alias: jmx.gc.minor_collection_count + CollectionTime: + metric_type: counter + alias: jmx.gc.minor_collection_time + - include: + domain: java.lang + type: GarbageCollector + name: G1 Young Generation + attribute: + CollectionCount: + metric_type: counter + alias: jmx.gc.minor_collection_count + CollectionTime: + metric_type: counter + alias: jmx.gc.minor_collection_time + # Old Gen Collectors (Major collections) + - include: + domain: java.lang + type: GarbageCollector + name: MarkSweepCompact + attribute: + CollectionCount: + metric_type: counter + alias: jmx.gc.major_collection_count + CollectionTime: + metric_type: counter + alias: jmx.gc.major_collection_time + - include: + domain: java.lang + type: GarbageCollector + name: PS MarkSweep + attribute: + CollectionCount: + metric_type: counter + alias: jmx.gc.major_collection_count + CollectionTime: + metric_type: counter + alias: jmx.gc.major_collection_time + - include: + domain: java.lang + type: GarbageCollector + name: ConcurrentMarkSweep + attribute: + CollectionCount: + metric_type: counter + alias: jmx.gc.major_collection_count + CollectionTime: + metric_type: counter + alias: jmx.gc.major_collection_time + - include: + domain: java.lang + type: GarbageCollector + name: G1 Mixed Generation + attribute: + CollectionCount: + metric_type: counter + alias: jmx.gc.major_collection_count + CollectionTime: + metric_type: counter + alias: jmx.gc.major_collection_time + # Deprecated metrics for pre Cassandra 3.0 versions compatibility. + # If you are using cassandra 2, the metrics below will be used, + # otherwise they will be ignored. + - include: + domain: org.apache.cassandra.metrics + type: ColumnFamily + bean_regex: + - .*keyspace=.* + name: + - ReadLatency + - WriteLatency + attribute: + - 75thPercentile + - 95thPercentile + - 99thPercentile + - OneMinuteRate + exclude: + keyspace: + - system + - system_auth + - system_distributed + - system_schema + - system_traces + - include: + domain: org.apache.cassandra.metrics + type: ColumnFamily + bean_regex: + - .*keyspace=.* + name: + - RangeLatency + - CasPrepareLatency + - CasProposeLatency + - CasCommitLatency + - ViewLockAcquireTime + - ViewReadTime + attribute: + - 75thPercentile + - 95thPercentile + - OneMinuteRate + exclude: + keyspace: + - system + - system_auth + - system_distributed + - system_schema + - system_traces + - include: + domain: org.apache.cassandra.metrics + type: ColumnFamily + bean_regex: + - .*keyspace=.* + name: + - SSTablesPerReadHistogram + - TombstoneScannedHistogram + - WaitingOnFreeMemtableSpace + attribute: + - 75thPercentile + - 95thPercentile + exclude: + keyspace: + - system + - system_auth + - system_distributed + - system_schema + - system_traces + - include: + domain: org.apache.cassandra.metrics + type: ColumnFamily + bean_regex: + - .*keyspace=.* + name: + - ColUpdateTimeDeltaHistogram + attribute: + - Min + - 75thPercentile + - 95thPercentile + exclude: + keyspace: + - system + - system_auth + - system_distributed + - system_schema + - system_traces + - include: + domain: org.apache.cassandra.metrics + type: ColumnFamily + bean_regex: + - .*keyspace=.* + name: + - BloomFilterFalseRatio + - CompressionRatio + - KeyCacheHitRate + - LiveSSTableCount + - MaxPartitionSize + - MeanPartitionSize + - MeanRowSize + - MaxRowSize + - PendingCompactions + - SnapshotsSize + attribute: + - Value + exclude: + keyspace: + - system + - system_auth + - system_distributed + - system_schema + - system_traces + - include: + domain: org.apache.cassandra.metrics + type: ColumnFamily + bean_regex: + - .*keyspace=.* + name: + - PendingFlushes + - LiveDiskSpaceUsed + - TotalDiskSpaceUsed + - RowCacheHitOutOfRange + - RowCacheHit + - RowCacheMiss + attribute: + - Count + exclude: + keyspace: + - system + - system_auth + - system_distributed + - system_schema + - system_traces + - include: + domain: org.apache.cassandra.db + type: ColumnFamilies + attribute: + DroppableTombstoneRatio: + alias: cassandra.db.droppable_tombstone_ratio diff --git a/dd-java-agent/agent-jmxfetch/src/main/resources/datadog/trace/agent/jmxfetch/metricconfigs/kafka.yaml b/dd-java-agent/agent-jmxfetch/src/main/resources/datadog/trace/agent/jmxfetch/metricconfigs/kafka.yaml new file mode 100644 index 0000000000..6bb326e570 --- /dev/null +++ b/dd-java-agent/agent-jmxfetch/src/main/resources/datadog/trace/agent/jmxfetch/metricconfigs/kafka.yaml @@ -0,0 +1,566 @@ +# Metrics collected by this check. You should not have to modify this. +jmx_metrics: + # + # Producers (only v0.8.2.x) + # + - include: + domain: 'kafka.producer' + bean_regex: 'kafka\.producer:type=ProducerRequestMetrics,name=ProducerRequestRateAndTimeMs,clientId=.*' + attribute: + Count: + metric_type: rate + alias: kafka.producer.request_rate + - include: + domain: 'kafka.producer' + bean_regex: 'kafka\.producer:type=ProducerRequestMetrics,name=ProducerRequestRateAndTimeMs,clientId=.*' + attribute: + Mean: + metric_type: gauge + alias: kafka.producer.request_latency_avg + - include: + domain: 'kafka.producer' + bean_regex: 'kafka\.producer:type=ProducerTopicMetrics,name=BytesPerSec,clientId=.*' + attribute: + Count: + metric_type: rate + alias: kafka.producer.bytes_out + - include: + domain: 'kafka.producer' + bean_regex: 'kafka\.producer:type=ProducerTopicMetrics,name=MessagesPerSec,clientId=.*' + attribute: + Count: + metric_type: rate + alias: kafka.producer.message_rate + + # + # Producers (v0.9.0.x to v0.10.2.x) + # + - include: + domain: 'kafka.producer' + bean_regex: 'kafka\.producer:type=producer-metrics,client-id=.*' + attribute: + response-rate: + metric_type: gauge + alias: kafka.producer.response_rate + - include: + domain: 'kafka.producer' + bean_regex: 'kafka\.producer:type=producer-metrics,client-id=.*' + attribute: + request-rate: + metric_type: gauge + alias: kafka.producer.request_rate + - include: + domain: 'kafka.producer' + bean_regex: 'kafka\.producer:type=producer-metrics,client-id=.*' + attribute: + request-latency-avg: + metric_type: gauge + alias: kafka.producer.request_latency_avg + - include: + domain: 'kafka.producer' + bean_regex: 'kafka\.producer:type=producer-metrics,client-id=.*' + attribute: + outgoing-byte-rate: + metric_type: gauge + alias: kafka.producer.bytes_out + - include: + domain: 'kafka.producer' + bean_regex: 'kafka\.producer:type=producer-metrics,client-id=.*' + attribute: + io-wait-time-ns-avg: + metric_type: gauge + alias: kafka.producer.io_wait + + - include: + domain: 'kafka.producer' + bean_regex: 'kafka.producer:type=producer-metrics,client-id=([-.\w]+)' + attribute: + - waiting-threads: + metric_type: gauge + alias: kafka.producer.waiting_threads + - buffer-total-bytes: + metric_type: gauge + alias: kafka.producer.buffer_bytes_total + - buffer-available-bytes: + metric_type: gauge + alias: kafka.producer.available_buffer_bytes + - bufferpool-wait-time: + metric_type: gauge + alias: kafka.producer.bufferpool_wait_time + - batch-size-avg: + metric_type: gauge + alias: kafka.producer.batch_size_avg + - batch-size-max: + metric_type: gauge + alias: kafka.producer.batch_size_max + - compression-rate-avg: + metric_type: gauge + alias: kafka.producer.compression_rate_avg + - record-queue-time-avg: + metric_type: gauge + alias: kafka.producer.record_queue_time_avg + - record-queue-time-max: + metric_type: gauge + alias: kafka.producer.record_queue_time_max + - request-latency-avg: + metric_type: gauge + alias: kafka.producer.request_latency_avg + - request-latency-max: + metric_type: gauge + alias: kafka.producer.request_latency_max + - record-send-rate: + metric_type: gauge + alias: kafka.producer.records_send_rate + - records-per-request-avg: + metric_type: gauge + alias: kafka.producer.records_per_request + - record-retry-rate: + metric_type: gauge + alias: kafka.producer.record_retry_rate + - record-error-rate: + metric_type: gauge + alias: kafka.producer.record_error_rate + - record-size-max: + metric_type: gauge + alias: kafka.producer.record_size_max + - record-size-avg: + metric_type: gauge + alias: kafka.producer.record_size_avg + - requests-in-flight: + metric_type: gauge + alias: kafka.producer.requests_in_flight + - metadata-age: + metric_type: gauge + alias: kafka.producer.metadata_age + - produce-throttle-time-max: + metric_type: gauge + alias: kafka.producer.throttle_time_max + - produce-throttle-time-avg: + metric_type: gauge + alias: kafka.producer.throttle_time_avg + + + # + # Producers: Per Topic Metrics + # + - include: + domain: 'kafka.producer' + bean_regex: 'kafka\.producer:type=producer-topic-metrics,client-id=(.*?),topic=(.*?)(?:,|$)' + attribute: + byte-rate: + metric_type: gauge + alias: kafka.producer.bytes_out + tags: + client: $1 + topic: $2 + - include: + domain: 'kafka.producer' + bean_regex: 'kafka\.producer:type=producer-topic-metrics,client-id=(.*?),topic=(.*?)(?:,|$)' + attribute: + record-send-rate: + metric_type: gauge + alias: kafka.producer.record_send_rate + tags: + client: $1 + topic: $2 + - include: + domain: 'kafka.producer' + bean_regex: 'kafka\.producer:type=producer-topic-metrics,client-id=(.*?),topic=(.*?)(?:,|$)' + attribute: + compression-rate: + metric_type: gauge + alias: kafka.producer.compression_rate + tags: + client: $1 + topic: $2 + - include: + domain: 'kafka.producer' + bean_regex: 'kafka\.producer:type=producer-topic-metrics,client-id=(.*?),topic=(.*?)(?:,|$)' + attribute: + record-retry-rate: + metric_type: gauge + alias: kafka.producer.record_retry_rate + tags: + client: $1 + topic: $2 + - include: + domain: 'kafka.producer' + bean_regex: 'kafka\.producer:type=producer-topic-metrics,client-id=(.*?),topic=(.*?)(?:,|$)' + attribute: + record-error-rate: + metric_type: gauge + alias: kafka.producer.record_error_rate + tags: + client: $1 + topic: $2 + + # + # Consumers (only v0.8.2.x) + # + - include: + domain: 'kafka.consumer' + bean_regex: 'kafka\.consumer:type=ConsumerFetcherManager,name=MaxLag,clientId=.*' + attribute: + Value: + metric_type: gauge + alias: kafka.consumer.max_lag + - include: + domain: 'kafka.consumer' + bean_regex: 'kafka\.consumer:type=ConsumerFetcherManager,name=MinFetchRate,clientId=.*' + attribute: + Value: + metric_type: gauge + alias: kafka.consumer.fetch_rate + - include: + domain: 'kafka.consumer' + bean_regex: 'kafka\.consumer:type=ConsumerTopicMetrics,name=BytesPerSec,clientId=.*' + attribute: + Count: + metric_type: rate + alias: kafka.consumer.bytes_in + - include: + domain: 'kafka.consumer' + bean_regex: 'kafka\.consumer:type=ConsumerTopicMetrics,name=MessagesPerSec,clientId=.*' + attribute: + Count: + metric_type: rate + alias: kafka.consumer.messages_in + - include: + # Offsets committed to ZooKeeper + domain: 'kafka.consumer' + bean_regex: 'kafka\.consumer:type=ZookeeperConsumerConnector,name=ZooKeeperCommitsPerSec,clientId=.*' + attribute: + Count: + metric_type: rate + alias: kafka.consumer.zookeeper_commits + - include: + # Offsets committed to Kafka + domain: 'kafka.consumer' + bean_regex: 'kafka\.consumer:type=ZookeeperConsumerConnector,name=KafkaCommitsPerSec,clientId=.*' + attribute: + Count: + metric_type: rate + alias: kafka.consumer.kafka_commits + + # + # Consumers (v0.9.0.x to v0.10.2.x) + # + - include: + domain: 'kafka.consumer' + bean_regex: 'kafka\.consumer:type=consumer-fetch-manager-metrics,client-id=.*' + attribute: + bytes-consumed-rate: + metric_type: gauge + alias: kafka.consumer.bytes_in + - include: + domain: 'kafka.consumer' + bean_regex: 'kafka\.consumer:type=consumer-fetch-manager-metrics,client-id=.*' + attribute: + records-consumed-rate: + metric_type: gauge + alias: kafka.consumer.messages_in + + # + # Consumers: Per Topic Metrics + # + + - include: + domain: 'kafka.consumer' + bean_regex: 'kafka\.consumer:type=consumer-fetch-manager-metrics,client-id=(.*?),topic=(.*?)(?:,|$)' + attribute: + fetch-size-avg: + metric_type: gauge + alias: kafka.consumer.fetch_size_avg + tags: + client: $1 + topic: $2 + - include: + domain: 'kafka.consumer' + bean_regex: 'kafka\.consumer:type=consumer-fetch-manager-metrics,client-id=(.*?),topic=(.*?)(?:,|$)' + attribute: + fetch-size-max: + metric_type: gauge + alias: kafka.consumer.fetch_size_max + tags: + client: $1 + topic: $2 + - include: + domain: 'kafka.consumer' + bean_regex: 'kafka\.consumer:type=consumer-fetch-manager-metrics,client-id=(.*?),topic=(.*?)(?:,|$)' + attribute: + bytes-consumed-rate: + metric_type: gauge + alias: kafka.consumer.bytes_consumed + tags: + client: $1 + topic: $2 + - include: + domain: 'kafka.consumer' + bean_regex: 'kafka\.consumer:type=consumer-fetch-manager-metrics,client-id=(.*?),topic=(.*?)(?:,|$)' + attribute: + records-per-request-avg: + metric_type: gauge + alias: kafka.consumer.records_per_request_avg + tags: + client: $1 + topic: $2 + - include: + domain: 'kafka.consumer' + bean_regex: 'kafka\.consumer:type=consumer-fetch-manager-metrics,client-id=(.*?),topic=(.*?)(?:,|$)' + attribute: + records-consumed-rate: + metric_type: gauge + alias: kafka.consumer.records_consumed + tags: + client: $1 + topic: $2 + + # + # Aggregate cluster stats + # + - include: + domain: 'kafka.server' + bean: 'kafka.server:type=BrokerTopicMetrics,name=BytesOutPerSec' + attribute: + Count: + metric_type: rate + alias: kafka.net.bytes_out.rate + - include: + domain: 'kafka.server' + bean: 'kafka.server:type=BrokerTopicMetrics,name=BytesInPerSec' + attribute: + Count: + metric_type: rate + alias: kafka.net.bytes_in.rate + - include: + domain: 'kafka.server' + bean: 'kafka.server:type=BrokerTopicMetrics,name=MessagesInPerSec' + attribute: + Count: + metric_type: rate + alias: kafka.messages_in.rate + - include: + domain: 'kafka.server' + bean: 'kafka.server:type=BrokerTopicMetrics,name=BytesRejectedPerSec' + attribute: + Count: + metric_type: rate + alias: kafka.net.bytes_rejected.rate + + # + # Per Topic Broker Stats + # + - include: + domain: '"kafka.server"' + bean_regex: '"kafka.server":type="BrokerTopicMetrics",name="(.*?)-MessagesInPerSec"' + attribute: + Count: + metric_type: rate + alias: kafka.topic.messages_in.rate + tags: + topic: $1 + + # + # Request timings + # + - include: + domain: 'kafka.server' + bean: 'kafka.server:type=BrokerTopicMetrics,name=FailedFetchRequestsPerSec' + attribute: + Count: + metric_type: rate + alias: kafka.request.fetch.failed.rate + - include: + domain: 'kafka.server' + bean: 'kafka.server:type=BrokerTopicMetrics,name=FailedProduceRequestsPerSec' + attribute: + Count: + metric_type: rate + alias: kafka.request.produce.failed.rate + - include: + domain: 'kafka.network' + bean: 'kafka.network:type=RequestMetrics,name=RequestsPerSec,request=Produce' + attribute: + Count: + metric_type: rate + alias: kafka.request.produce.rate + - include: + domain: 'kafka.network' + bean: 'kafka.network:type=RequestMetrics,name=TotalTimeMs,request=Produce' + attribute: + Mean: + metric_type: gauge + alias: kafka.request.produce.time.avg + 99thPercentile: + metric_type: gauge + alias: kafka.request.produce.time.99percentile + - include: + domain: 'kafka.network' + bean: 'kafka.network:type=RequestMetrics,name=RequestsPerSec,request=FetchConsumer' + attribute: + Count: + metric_type: rate + alias: kafka.request.fetch_consumer.rate + - include: + domain: 'kafka.network' + bean: 'kafka.network:type=RequestMetrics,name=RequestsPerSec,request=FetchFollower' + attribute: + Count: + metric_type: rate + alias: kafka.request.fetch_follower.rate + - include: + domain: 'kafka.network' + bean: 'kafka.network:type=RequestMetrics,name=TotalTimeMs,request=FetchConsumer' + attribute: + Mean: + metric_type: gauge + alias: kafka.request.fetch_consumer.time.avg + 99thPercentile: + metric_type: gauge + alias: kafka.request.fetch_consumer.time.99percentile + - include: + domain: 'kafka.network' + bean: 'kafka.network:type=RequestMetrics,name=TotalTimeMs,request=FetchFollower' + attribute: + Mean: + metric_type: gauge + alias: kafka.request.fetch_follower.time.avg + 99thPercentile: + metric_type: gauge + alias: kafka.request.fetch_follower.time.99percentile + - include: + domain: 'kafka.network' + bean: 'kafka.network:type=RequestMetrics,name=TotalTimeMs,request=UpdateMetadata' + attribute: + Mean: + metric_type: gauge + alias: kafka.request.update_metadata.time.avg + 99thPercentile: + metric_type: gauge + alias: kafka.request.update_metadata.time.99percentile + - include: + domain: 'kafka.network' + bean: 'kafka.network:type=RequestMetrics,name=TotalTimeMs,request=Metadata' + attribute: + Mean: + metric_type: gauge + alias: kafka.request.metadata.time.avg + 99thPercentile: + metric_type: gauge + alias: kafka.request.metadata.time.99percentile + - include: + domain: 'kafka.network' + bean: 'kafka.network:type=RequestMetrics,name=TotalTimeMs,request=Offsets' + attribute: + Mean: + metric_type: gauge + alias: kafka.request.offsets.time.avg + 99thPercentile: + metric_type: gauge + alias: kafka.request.offsets.time.99percentile + - include: + domain: 'kafka.server' + bean: 'kafka.server:type=KafkaRequestHandlerPool,name=RequestHandlerAvgIdlePercent' + attribute: + OneMinuteRate: + metric_type: gauge + alias: kafka.request.handler.avg.idle.pct.rate + - include: + domain: 'kafka.server' + bean: 'kafka.server:type=ProducerRequestPurgatory,name=PurgatorySize' + attribute: + Value: + metric_type: gauge + alias: kafka.request.producer_request_purgatory.size + - include: + domain: 'kafka.server' + bean: 'kafka.server:type=FetchRequestPurgatory,name=PurgatorySize' + attribute: + Value: + metric_type: gauge + alias: kafka.request.fetch_request_purgatory.size + + # + # Replication stats + # + - include: + domain: 'kafka.server' + bean: 'kafka.server:type=ReplicaManager,name=UnderReplicatedPartitions' + attribute: + Value: + metric_type: gauge + alias: kafka.replication.under_replicated_partitions + - include: + domain: 'kafka.server' + bean: 'kafka.server:type=ReplicaManager,name=IsrShrinksPerSec' + attribute: + Count: + metric_type: rate + alias: kafka.replication.isr_shrinks.rate + - include: + domain: 'kafka.server' + bean: 'kafka.server:type=ReplicaManager,name=IsrExpandsPerSec' + attribute: + Count: + metric_type: rate + alias: kafka.replication.isr_expands.rate + - include: + domain: 'kafka.controller' + bean: 'kafka.controller:type=ControllerStats,name=LeaderElectionRateAndTimeMs' + attribute: + Count: + metric_type: rate + alias: kafka.replication.leader_elections.rate + - include: + domain: 'kafka.controller' + bean: 'kafka.controller:type=ControllerStats,name=UncleanLeaderElectionsPerSec' + attribute: + Count: + metric_type: rate + alias: kafka.replication.unclean_leader_elections.rate + - include: + domain: 'kafka.controller' + bean: 'kafka.controller:type=KafkaController,name=OfflinePartitionsCount' + attribute: + Value: + metric_type: gauge + alias: kafka.replication.offline_partitions_count + - include: + domain: 'kafka.controller' + bean: 'kafka.controller:type=KafkaController,name=ActiveControllerCount' + attribute: + Value: + metric_type: gauge + alias: kafka.replication.active_controller_count + - include: + domain: 'kafka.server' + bean: 'kafka.server:type=ReplicaManager,name=PartitionCount' + attribute: + Value: + metric_type: gauge + alias: kafka.replication.partition_count + - include: + domain: 'kafka.server' + bean: 'kafka.server:type=ReplicaManager,name=LeaderCount' + attribute: + Value: + metric_type: gauge + alias: kafka.replication.leader_count + - include: + domain: 'kafka.server' + bean: 'kafka.server:type=ReplicaFetcherManager,name=MaxLag,clientId=Replica' + attribute: + Value: + metric_type: gauge + alias: kafka.replication.max_lag + + # + # Log flush stats + # + - include: + domain: 'kafka.log' + bean: 'kafka.log:type=LogFlushStats,name=LogFlushRateAndTimeMs' + attribute: + Count: + metric_type: rate + alias: kafka.log.flush_rate.rate diff --git a/dd-java-agent/agent-jmxfetch/src/main/resources/datadog/trace/agent/jmxfetch/metricconfigs/solr.yaml b/dd-java-agent/agent-jmxfetch/src/main/resources/datadog/trace/agent/jmxfetch/metricconfigs/solr.yaml new file mode 100644 index 0000000000..3fa966c5a0 --- /dev/null +++ b/dd-java-agent/agent-jmxfetch/src/main/resources/datadog/trace/agent/jmxfetch/metricconfigs/solr.yaml @@ -0,0 +1,200 @@ +# Default metrics collected by this check. You should not have to modify this. +jmx_metrics: + + # Solr versions >= 7.0 + - include: + category: SEARCHER + name: + - numDocs + scope: + - searcher + attribute: + Value: + alias: solr.searcher.numdocs + metric_type: gauge + + - include: + category: SEARCHER + name: maxDoc + scope: + - searcher + attribute: + Value: + alias: solr.searcher.maxdocs + metric_type: gauge + + - include: + category: SEARCHER + name: warmupTime + scope: + - searcher + attribute: + Value: + alias: solr.searcher.warmup + metric_type: gauge + + - include: + category: CACHE + name: documentCache + scope: + - searcher + attribute: + cumulative_lookups: + alias: solr.document_cache.lookups + metric_type: counter + cumulative_hits: + alias: solr.document_cache.hits + metric_type: counter + cumulative_inserts: + alias: solr.document_cache.inserts + metric_type: counter + cumulative_evictions: + alias: solr.document_cache.evictions + metric_type: counter + + - include: + category: CACHE + name: queryResultCache + scope: + - searcher + attribute: + cumulative_lookups: + alias: solr.query_result_cache.lookups + metric_type: counter + cumulative_hits: + alias: solr.query_result_cache.hits + metric_type: counter + cumulative_inserts: + alias: solr.query_result_cache.inserts + metric_type: counter + cumulative_evictions: + alias: solr.query_result_cache.evictions + metric_type: counter + + - include: + category: CACHE + name: filterCache + scope: + - searcher + attribute: + cumulative_lookups: + alias: solr.filter_cache.lookups + metric_type: counter + cumulative_hits: + alias: solr.filter_cache.hits + metric_type: counter + cumulative_inserts: + alias: solr.filter_cache.inserts + metric_type: counter + cumulative_evictions: + alias: solr.filter_cache.evictions + metric_type: counter + + - include: + category: QUERY + name: requests + attribute: + Count: + alias: solr.search_handler.requests + metric_type: counter + + - include: + category: QUERY + name: timeouts + attribute: + Count: + alias: solr.search_handler.timeouts + metric_type: counter + + - include: + category: QUERY + name: errors + attribute: + Count: + alias: solr.search_handler.errors + metric_type: counter + + - include: + category: QUERY + name: totalTime + attribute: + Count: + alias: solr.search_handler.time + metric_type: counter + + - include: + category: QUERY + name: requestTimes + attribute: + avgRequestsPerSecond: + alias: solr.search_handler.avg_requests_per_sec + metric_type: gauge + avgTimePerRequest: + alias: solr.search_handler.avg_time_per_req + metric_type: gauge + + + # Solr versions < 7.0 + - include: + type: searcher + attribute: + maxDoc: + alias: solr.searcher.maxdoc + metric_type: gauge + numDocs: + alias: solr.searcher.numdocs + metric_type: gauge + warmupTime: + alias: solr.searcher.warmup + metric_type: gauge + - include: + id: org.apache.solr.search.FastLRUCache + attribute: + cumulative_lookups: + alias: solr.cache.lookups + metric_type: counter + cumulative_hits: + alias: solr.cache.hits + metric_type: counter + cumulative_inserts: + alias: solr.cache.inserts + metric_type: counter + cumulative_evictions: + alias: solr.cache.evictions + metric_type: counter + - include: + id: org.apache.solr.search.LRUCache + attribute: + cumulative_lookups: + alias: solr.cache.lookups + metric_type: counter + cumulative_hits: + alias: solr.cache.hits + metric_type: counter + cumulative_inserts: + alias: solr.cache.inserts + metric_type: counter + cumulative_evictions: + alias: solr.cache.evictions + metric_type: counter + - include: + id: org.apache.solr.handler.component.SearchHandler + attribute: + errors: + alias: solr.search_handler.errors + metric_type: counter + requests: + alias: solr.search_handler.requests + metric_type: counter + timeouts: + alias: solr.search_handler.timeouts + metric_type: counter + totalTime: + alias: solr.search_handler.time + metric_type: counter + avgTimePerRequest: + alias: solr.search_handler.avg_time_per_req + metric_type: gauge + avgRequestsPerSecond: + alias: solr.search_handler.avg_requests_per_sec + metric_type: gauge diff --git a/dd-java-agent/agent-jmxfetch/src/main/resources/datadog/trace/agent/jmxfetch/metricconfigs/tomcat.yaml b/dd-java-agent/agent-jmxfetch/src/main/resources/datadog/trace/agent/jmxfetch/metricconfigs/tomcat.yaml new file mode 100644 index 0000000000..370cb90421 --- /dev/null +++ b/dd-java-agent/agent-jmxfetch/src/main/resources/datadog/trace/agent/jmxfetch/metricconfigs/tomcat.yaml @@ -0,0 +1,65 @@ +# Default metrics collected by this check. You should not have to modify this. +jmx_metrics: + - include: + type: ThreadPool + attribute: + maxThreads: + alias: tomcat.threads.max + metric_type: gauge + currentThreadCount: + alias: tomcat.threads.count + metric_type: gauge + currentThreadsBusy: + alias: tomcat.threads.busy + metric_type: gauge + - include: + type: GlobalRequestProcessor + attribute: + bytesSent: + alias: tomcat.bytes_sent + metric_type: counter + bytesReceived: + alias: tomcat.bytes_rcvd + metric_type: counter + errorCount: + alias: tomcat.error_count + metric_type: counter + requestCount: + alias: tomcat.request_count + metric_type: counter + maxTime: + alias: tomcat.max_time + metric_type: gauge + processingTime: + alias: tomcat.processing_time + metric_type: counter + - include: + j2eeType: Servlet + attribute: + processingTime: + alias: tomcat.servlet.processing_time + metric_type: counter + errorCount: + alias: tomcat.servlet.error_count + metric_type: counter + requestCount: + alias: tomcat.servlet.request_count + metric_type: counter + - include: + type: Cache + attribute: + accessCount: + alias: tomcat.cache.access_count + metric_type: counter + hitsCounts: + alias: tomcat.cache.hits_count + metric_type: counter + - include: + type: JspMonitor + attribute: + jspCount: + alias: tomcat.jsp.count + metric_type: counter + jspReloadCount: + alias: tomcat.jsp.reload_count + metric_type: counter From 249951aa7a6e816daa8c102089e87849065f9930 Mon Sep 17 00:00:00 2001 From: Tyler Benson Date: Wed, 2 Jan 2019 15:30:55 -0800 Subject: [PATCH 2/4] Add README --- .../main/resources/datadog/trace/agent/jmxfetch/README | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 dd-java-agent/agent-jmxfetch/src/main/resources/datadog/trace/agent/jmxfetch/README diff --git a/dd-java-agent/agent-jmxfetch/src/main/resources/datadog/trace/agent/jmxfetch/README b/dd-java-agent/agent-jmxfetch/src/main/resources/datadog/trace/agent/jmxfetch/README new file mode 100644 index 0000000000..ebd3b35089 --- /dev/null +++ b/dd-java-agent/agent-jmxfetch/src/main/resources/datadog/trace/agent/jmxfetch/README @@ -0,0 +1,8 @@ +# Metric Configs + +These files are copied directly from [integrations-core](https://github.com/DataDog/integrations-core/search?q=jmx_metrics&unscoped_q=jmx_metrics) +where the only modification is updating the file name. + +These will be bundled in `dd-java-agent.jar`. Due to limitations in Java, it is non-trivial +to get all these files from within the jar without knowing their names. +Consequently, we list out each integration in `metricconfigs.txt` so the agent can reference them. From f63d40fe5591f8591eee0567dc38b22986c2454a Mon Sep 17 00:00:00 2001 From: Tyler Benson Date: Tue, 8 Jan 2019 09:01:48 -0800 Subject: [PATCH 3/4] Use git submodules to load metrics.yaml files --- .gitmodules | 3 + .../agent-jmxfetch/agent-jmxfetch.gradle | 16 + .../agent-jmxfetch/copy-metric-configs.sh | 51 ++ .../agent-jmxfetch/integrations-core | 1 + .../datadog/trace/agent/jmxfetch/README | 6 +- .../trace/agent/jmxfetch/metricconfigs.txt | 5 - .../jmxfetch/metricconfigs/activemq.yaml | 54 -- .../jmxfetch/metricconfigs/cassandra.yaml | 427 ------------- .../agent/jmxfetch/metricconfigs/kafka.yaml | 566 ------------------ .../agent/jmxfetch/metricconfigs/solr.yaml | 200 ------- .../agent/jmxfetch/metricconfigs/tomcat.yaml | 65 -- 11 files changed, 74 insertions(+), 1320 deletions(-) create mode 100644 .gitmodules create mode 100755 dd-java-agent/agent-jmxfetch/copy-metric-configs.sh create mode 160000 dd-java-agent/agent-jmxfetch/integrations-core delete mode 100644 dd-java-agent/agent-jmxfetch/src/main/resources/datadog/trace/agent/jmxfetch/metricconfigs.txt delete mode 100644 dd-java-agent/agent-jmxfetch/src/main/resources/datadog/trace/agent/jmxfetch/metricconfigs/activemq.yaml delete mode 100644 dd-java-agent/agent-jmxfetch/src/main/resources/datadog/trace/agent/jmxfetch/metricconfigs/cassandra.yaml delete mode 100644 dd-java-agent/agent-jmxfetch/src/main/resources/datadog/trace/agent/jmxfetch/metricconfigs/kafka.yaml delete mode 100644 dd-java-agent/agent-jmxfetch/src/main/resources/datadog/trace/agent/jmxfetch/metricconfigs/solr.yaml delete mode 100644 dd-java-agent/agent-jmxfetch/src/main/resources/datadog/trace/agent/jmxfetch/metricconfigs/tomcat.yaml diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000000..27fc232a99 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "dd-java-agent/agent-jmxfetch/integrations-core"] + path = dd-java-agent/agent-jmxfetch/integrations-core + url = https://github.com/DataDog/integrations-core.git diff --git a/dd-java-agent/agent-jmxfetch/agent-jmxfetch.gradle b/dd-java-agent/agent-jmxfetch/agent-jmxfetch.gradle index 567be7e0f8..5bb89f4109 100644 --- a/dd-java-agent/agent-jmxfetch/agent-jmxfetch.gradle +++ b/dd-java-agent/agent-jmxfetch/agent-jmxfetch.gradle @@ -27,3 +27,19 @@ shadowJar { jar { classifier = 'unbundled' } + +task submodulesUpdate(type:Exec) { + group 'Build Setup' + description 'Initializes and updates integrations-core git submodule' + commandLine 'git', 'submodule', 'update', '--init', 'integrations-core' +} + +task copyMetricConfigs(type:Exec) { + group 'Build Setup' + description 'Copy metrics.yaml files from integrations-core into resources' + commandLine './copy-metric-configs.sh', 'integrations-core', sourceSets.main.output.resourcesDir +} + +processResources.dependsOn copyMetricConfigs +copyMetricConfigs.dependsOn submodulesUpdate +// Runs the copy task before processResources, so duplicate files in the local repo would overwrite imported ones. diff --git a/dd-java-agent/agent-jmxfetch/copy-metric-configs.sh b/dd-java-agent/agent-jmxfetch/copy-metric-configs.sh new file mode 100755 index 0000000000..664e277319 --- /dev/null +++ b/dd-java-agent/agent-jmxfetch/copy-metric-configs.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash + +# Debug mode: +# set -x + +function print_usage() { + echo "Usage: $0 search_directory build_resources_output_directory" +} + +search_directory=$1 +if [ ! -d "$search_directory" ]; then + echo "Must specify a valid search_directory" + print_usage + exit 1 +fi + +build_resources_output_directory=$2 +if [ ! -d "$build_resources_output_directory" ]; then + echo "Must specify a valid build_resources_output_directory" + print_usage + exit 1 +fi +# Add the full package path. +build_resources_output_directory="$build_resources_output_directory/datadog/trace/agent/jmxfetch" + +# Find all the metrics.yaml files containing "jmx_metrics:" +metrics_files=$(grep --include=metrics.yaml -rwl $search_directory -e 'jmx_metrics:') + +if [ -z "$metrics_files" ]; then + echo "No metrics.yaml files with jmx_metrics blocks found." + print_usage + exit 1 +fi + +# reset file and ensure directories exists +> $build_resources_output_directory/metricconfigs.txt +mkdir -p $build_resources_output_directory/metricconfigs + +for input_file in $metrics_files ; do + # generate new name based on integration folder name which should look like this: + # integrations-core//datadog_checks//data/metrics.yaml + output_file=$(echo "$input_file" | awk -F/ '{print $2}') + + # save file name in metricconfigs.txt + echo "$output_file.yaml" >> $build_resources_output_directory/metricconfigs.txt + + # copy to output location + output_file="$build_resources_output_directory/metricconfigs/$output_file.yaml" + cp $input_file $output_file +done + diff --git a/dd-java-agent/agent-jmxfetch/integrations-core b/dd-java-agent/agent-jmxfetch/integrations-core new file mode 160000 index 0000000000..3e38b4e75e --- /dev/null +++ b/dd-java-agent/agent-jmxfetch/integrations-core @@ -0,0 +1 @@ +Subproject commit 3e38b4e75edcee3ca84f022ea50240b0fc0537f2 diff --git a/dd-java-agent/agent-jmxfetch/src/main/resources/datadog/trace/agent/jmxfetch/README b/dd-java-agent/agent-jmxfetch/src/main/resources/datadog/trace/agent/jmxfetch/README index ebd3b35089..557fc3337c 100644 --- a/dd-java-agent/agent-jmxfetch/src/main/resources/datadog/trace/agent/jmxfetch/README +++ b/dd-java-agent/agent-jmxfetch/src/main/resources/datadog/trace/agent/jmxfetch/README @@ -1,8 +1,8 @@ # Metric Configs -These files are copied directly from [integrations-core](https://github.com/DataDog/integrations-core/search?q=jmx_metrics&unscoped_q=jmx_metrics) -where the only modification is updating the file name. +Files from [integrations-core](https://github.com/DataDog/integrations-core/search?q=jmx_metrics&unscoped_q=jmx_metrics) +are copied here at build time by the `copy-metric-configs.sh` script which is called by gradle after initializing the submodule. -These will be bundled in `dd-java-agent.jar`. Due to limitations in Java, it is non-trivial +These are then bundled in `dd-java-agent.jar`. Due to limitations in Java, it is non-trivial to get all these files from within the jar without knowing their names. Consequently, we list out each integration in `metricconfigs.txt` so the agent can reference them. diff --git a/dd-java-agent/agent-jmxfetch/src/main/resources/datadog/trace/agent/jmxfetch/metricconfigs.txt b/dd-java-agent/agent-jmxfetch/src/main/resources/datadog/trace/agent/jmxfetch/metricconfigs.txt deleted file mode 100644 index 521c7d2204..0000000000 --- a/dd-java-agent/agent-jmxfetch/src/main/resources/datadog/trace/agent/jmxfetch/metricconfigs.txt +++ /dev/null @@ -1,5 +0,0 @@ -activemq.yaml -cassandra.yaml -kafka.yaml -solr.yaml -tomcat.yaml diff --git a/dd-java-agent/agent-jmxfetch/src/main/resources/datadog/trace/agent/jmxfetch/metricconfigs/activemq.yaml b/dd-java-agent/agent-jmxfetch/src/main/resources/datadog/trace/agent/jmxfetch/metricconfigs/activemq.yaml deleted file mode 100644 index d0735950ba..0000000000 --- a/dd-java-agent/agent-jmxfetch/src/main/resources/datadog/trace/agent/jmxfetch/metricconfigs/activemq.yaml +++ /dev/null @@ -1,54 +0,0 @@ -# Default metrics collected by this check. You should not have to modify this. -jmx_metrics: -- include: - destinationType: Queue - attribute: - AverageEnqueueTime: - alias: activemq.queue.avg_enqueue_time - metric_type: gauge - ConsumerCount: - alias: activemq.queue.consumer_count - metric_type: gauge - ProducerCount: - alias: activemq.queue.producer_count - metric_type: gauge - MaxEnqueueTime: - alias: activemq.queue.max_enqueue_time - metric_type: gauge - MinEnqueueTime: - alias: activemq.queue.min_enqueue_time - metric_type: gauge - MemoryPercentUsage: - alias: activemq.queue.memory_pct - metric_type: gauge - QueueSize: - alias: activemq.queue.size - metric_type: gauge - DequeueCount: - alias: activemq.queue.dequeue_count - metric_type: counter - DispatchCount: - alias: activemq.queue.dispatch_count - metric_type: counter - EnqueueCount: - alias: activemq.queue.enqueue_count - metric_type: counter - ExpiredCount: - alias: activemq.queue.expired_count - metric_type: counter - InFlightCount: - alias: activemq.queue.in_flight_count - metric_type: counter - -- include: - type: Broker - attribute: - StorePercentUsage: - alias: activemq.broker.store_pct - metric_type: gauge - TempPercentUsage: - alias: activemq.broker.temp_pct - metric_type: gauge - MemoryPercentUsage: - alias: activemq.broker.memory_pct - metric_type: gauge diff --git a/dd-java-agent/agent-jmxfetch/src/main/resources/datadog/trace/agent/jmxfetch/metricconfigs/cassandra.yaml b/dd-java-agent/agent-jmxfetch/src/main/resources/datadog/trace/agent/jmxfetch/metricconfigs/cassandra.yaml deleted file mode 100644 index a94c31374d..0000000000 --- a/dd-java-agent/agent-jmxfetch/src/main/resources/datadog/trace/agent/jmxfetch/metricconfigs/cassandra.yaml +++ /dev/null @@ -1,427 +0,0 @@ -# Default metrics collected by this check. You should not have to modify this. -jmx_metrics: - - include: - domain: org.apache.cassandra.metrics - type: ClientRequest - name: - - Latency - attribute: - - 75thPercentile - - 95thPercentile - - OneMinuteRate - - include: - domain: org.apache.cassandra.metrics - type: DroppedMessage - name: - - Dropped - attribute: - - OneMinuteRate - - include: - domain: org.apache.cassandra.metrics - type: ThreadPools - scope: - - MutationStage - - CounterMutationStage - - ReadStage - - ViewMutationStage - name: - - PendingTasks - - CurrentlyBlockedTasks - path: - - request - - include: - domain: org.apache.cassandra.metrics - type: ThreadPools - scope: - - MemtableFlushWriter - - HintsDispatcher - - MemtablePostFlush - - MigrationStage - - MiscStage - - SecondaryIndexManagement - name: - - PendingTasks - - CurrentlyBlockedTasks - path: - - internal - - include: - domain: org.apache.cassandra.metrics - type: Storage - name: - - Load - - Exceptions - - include: - domain: org.apache.cassandra.metrics - type: Table - bean_regex: - - .*keyspace=.* - name: - - ReadLatency - - WriteLatency - attribute: - - 75thPercentile - - 95thPercentile - - 99thPercentile - - OneMinuteRate - exclude: - keyspace: - - system - - system_auth - - system_distributed - - system_schema - - system_traces - - include: - domain: org.apache.cassandra.metrics - type: Table - bean_regex: - - .*keyspace=.* - name: - - RangeLatency - - CasPrepareLatency - - CasProposeLatency - - CasCommitLatency - - ViewLockAcquireTime - - ViewReadTime - attribute: - - 75thPercentile - - 95thPercentile - - OneMinuteRate - exclude: - keyspace: - - system - - system_auth - - system_distributed - - system_schema - - system_traces - - include: - domain: org.apache.cassandra.metrics - type: Table - bean_regex: - - .*keyspace=.* - name: - - SSTablesPerReadHistogram - - TombstoneScannedHistogram - - WaitingOnFreeMemtableSpace - attribute: - - 75thPercentile - - 95thPercentile - exclude: - keyspace: - - system - - system_auth - - system_distributed - - system_schema - - system_traces - - include: - domain: org.apache.cassandra.metrics - type: Table - bean_regex: - - .*keyspace=.* - name: - - ColUpdateTimeDeltaHistogram - attribute: - - Min - - 75thPercentile - - 95thPercentile - exclude: - keyspace: - - system - - system_auth - - system_distributed - - system_schema - - system_traces - - include: - domain: org.apache.cassandra.metrics - type: Table - bean_regex: - - .*keyspace=.* - name: - - BloomFilterFalseRatio - - CompressionRatio - - KeyCacheHitRate - - LiveSSTableCount - - MaxPartitionSize - - MeanPartitionSize - - MeanRowSize - - MaxRowSize - - PendingCompactions - - SnapshotsSize - attribute: - - Value - exclude: - keyspace: - - system - - system_auth - - system_distributed - - system_schema - - system_traces - - include: - domain: org.apache.cassandra.metrics - type: Table - bean_regex: - - .*keyspace=.* - name: - - CompactionBytesWritten - - BytesFlushed - - PendingFlushes - - LiveDiskSpaceUsed - - TotalDiskSpaceUsed - - RowCacheHitOutOfRange - - RowCacheHit - - RowCacheMiss - attribute: - - Count - exclude: - keyspace: - - system - - system_auth - - system_distributed - - system_schema - - system_traces - - include: - domain: org.apache.cassandra.metrics - type: Cache - scope: KeyCache - name: - - HitRate - attribute: - - Count - - include: - domain: org.apache.cassandra.metrics - type: CommitLog - name: - - PendingTasks - - TotalCommitLogSize - attribute: - - Value - - include: - domain: org.apache.cassandra.db - type: Tables - attribute: - DroppableTombstoneRatio: - alias: cassandra.db.droppable_tombstone_ratio - # Young Gen Collectors (Minor Collections) - - include: - domain: java.lang - type: GarbageCollector - name: Copy - attribute: - CollectionCount: - metric_type: counter - alias: jmx.gc.minor_collection_count - CollectionTime: - metric_type: counter - alias: jmx.gc.minor_collection_time - - include: - domain: java.lang - type: GarbageCollector - name: PS Scavenge - attribute: - CollectionCount: - metric_type: counter - alias: jmx.gc.minor_collection_count - CollectionTime: - metric_type: counter - alias: jmx.gc.minor_collection_time - - include: - domain: java.lang - type: GarbageCollector - name: ParNew - attribute: - CollectionCount: - metric_type: counter - alias: jmx.gc.minor_collection_count - CollectionTime: - metric_type: counter - alias: jmx.gc.minor_collection_time - - include: - domain: java.lang - type: GarbageCollector - name: G1 Young Generation - attribute: - CollectionCount: - metric_type: counter - alias: jmx.gc.minor_collection_count - CollectionTime: - metric_type: counter - alias: jmx.gc.minor_collection_time - # Old Gen Collectors (Major collections) - - include: - domain: java.lang - type: GarbageCollector - name: MarkSweepCompact - attribute: - CollectionCount: - metric_type: counter - alias: jmx.gc.major_collection_count - CollectionTime: - metric_type: counter - alias: jmx.gc.major_collection_time - - include: - domain: java.lang - type: GarbageCollector - name: PS MarkSweep - attribute: - CollectionCount: - metric_type: counter - alias: jmx.gc.major_collection_count - CollectionTime: - metric_type: counter - alias: jmx.gc.major_collection_time - - include: - domain: java.lang - type: GarbageCollector - name: ConcurrentMarkSweep - attribute: - CollectionCount: - metric_type: counter - alias: jmx.gc.major_collection_count - CollectionTime: - metric_type: counter - alias: jmx.gc.major_collection_time - - include: - domain: java.lang - type: GarbageCollector - name: G1 Mixed Generation - attribute: - CollectionCount: - metric_type: counter - alias: jmx.gc.major_collection_count - CollectionTime: - metric_type: counter - alias: jmx.gc.major_collection_time - # Deprecated metrics for pre Cassandra 3.0 versions compatibility. - # If you are using cassandra 2, the metrics below will be used, - # otherwise they will be ignored. - - include: - domain: org.apache.cassandra.metrics - type: ColumnFamily - bean_regex: - - .*keyspace=.* - name: - - ReadLatency - - WriteLatency - attribute: - - 75thPercentile - - 95thPercentile - - 99thPercentile - - OneMinuteRate - exclude: - keyspace: - - system - - system_auth - - system_distributed - - system_schema - - system_traces - - include: - domain: org.apache.cassandra.metrics - type: ColumnFamily - bean_regex: - - .*keyspace=.* - name: - - RangeLatency - - CasPrepareLatency - - CasProposeLatency - - CasCommitLatency - - ViewLockAcquireTime - - ViewReadTime - attribute: - - 75thPercentile - - 95thPercentile - - OneMinuteRate - exclude: - keyspace: - - system - - system_auth - - system_distributed - - system_schema - - system_traces - - include: - domain: org.apache.cassandra.metrics - type: ColumnFamily - bean_regex: - - .*keyspace=.* - name: - - SSTablesPerReadHistogram - - TombstoneScannedHistogram - - WaitingOnFreeMemtableSpace - attribute: - - 75thPercentile - - 95thPercentile - exclude: - keyspace: - - system - - system_auth - - system_distributed - - system_schema - - system_traces - - include: - domain: org.apache.cassandra.metrics - type: ColumnFamily - bean_regex: - - .*keyspace=.* - name: - - ColUpdateTimeDeltaHistogram - attribute: - - Min - - 75thPercentile - - 95thPercentile - exclude: - keyspace: - - system - - system_auth - - system_distributed - - system_schema - - system_traces - - include: - domain: org.apache.cassandra.metrics - type: ColumnFamily - bean_regex: - - .*keyspace=.* - name: - - BloomFilterFalseRatio - - CompressionRatio - - KeyCacheHitRate - - LiveSSTableCount - - MaxPartitionSize - - MeanPartitionSize - - MeanRowSize - - MaxRowSize - - PendingCompactions - - SnapshotsSize - attribute: - - Value - exclude: - keyspace: - - system - - system_auth - - system_distributed - - system_schema - - system_traces - - include: - domain: org.apache.cassandra.metrics - type: ColumnFamily - bean_regex: - - .*keyspace=.* - name: - - PendingFlushes - - LiveDiskSpaceUsed - - TotalDiskSpaceUsed - - RowCacheHitOutOfRange - - RowCacheHit - - RowCacheMiss - attribute: - - Count - exclude: - keyspace: - - system - - system_auth - - system_distributed - - system_schema - - system_traces - - include: - domain: org.apache.cassandra.db - type: ColumnFamilies - attribute: - DroppableTombstoneRatio: - alias: cassandra.db.droppable_tombstone_ratio diff --git a/dd-java-agent/agent-jmxfetch/src/main/resources/datadog/trace/agent/jmxfetch/metricconfigs/kafka.yaml b/dd-java-agent/agent-jmxfetch/src/main/resources/datadog/trace/agent/jmxfetch/metricconfigs/kafka.yaml deleted file mode 100644 index 6bb326e570..0000000000 --- a/dd-java-agent/agent-jmxfetch/src/main/resources/datadog/trace/agent/jmxfetch/metricconfigs/kafka.yaml +++ /dev/null @@ -1,566 +0,0 @@ -# Metrics collected by this check. You should not have to modify this. -jmx_metrics: - # - # Producers (only v0.8.2.x) - # - - include: - domain: 'kafka.producer' - bean_regex: 'kafka\.producer:type=ProducerRequestMetrics,name=ProducerRequestRateAndTimeMs,clientId=.*' - attribute: - Count: - metric_type: rate - alias: kafka.producer.request_rate - - include: - domain: 'kafka.producer' - bean_regex: 'kafka\.producer:type=ProducerRequestMetrics,name=ProducerRequestRateAndTimeMs,clientId=.*' - attribute: - Mean: - metric_type: gauge - alias: kafka.producer.request_latency_avg - - include: - domain: 'kafka.producer' - bean_regex: 'kafka\.producer:type=ProducerTopicMetrics,name=BytesPerSec,clientId=.*' - attribute: - Count: - metric_type: rate - alias: kafka.producer.bytes_out - - include: - domain: 'kafka.producer' - bean_regex: 'kafka\.producer:type=ProducerTopicMetrics,name=MessagesPerSec,clientId=.*' - attribute: - Count: - metric_type: rate - alias: kafka.producer.message_rate - - # - # Producers (v0.9.0.x to v0.10.2.x) - # - - include: - domain: 'kafka.producer' - bean_regex: 'kafka\.producer:type=producer-metrics,client-id=.*' - attribute: - response-rate: - metric_type: gauge - alias: kafka.producer.response_rate - - include: - domain: 'kafka.producer' - bean_regex: 'kafka\.producer:type=producer-metrics,client-id=.*' - attribute: - request-rate: - metric_type: gauge - alias: kafka.producer.request_rate - - include: - domain: 'kafka.producer' - bean_regex: 'kafka\.producer:type=producer-metrics,client-id=.*' - attribute: - request-latency-avg: - metric_type: gauge - alias: kafka.producer.request_latency_avg - - include: - domain: 'kafka.producer' - bean_regex: 'kafka\.producer:type=producer-metrics,client-id=.*' - attribute: - outgoing-byte-rate: - metric_type: gauge - alias: kafka.producer.bytes_out - - include: - domain: 'kafka.producer' - bean_regex: 'kafka\.producer:type=producer-metrics,client-id=.*' - attribute: - io-wait-time-ns-avg: - metric_type: gauge - alias: kafka.producer.io_wait - - - include: - domain: 'kafka.producer' - bean_regex: 'kafka.producer:type=producer-metrics,client-id=([-.\w]+)' - attribute: - - waiting-threads: - metric_type: gauge - alias: kafka.producer.waiting_threads - - buffer-total-bytes: - metric_type: gauge - alias: kafka.producer.buffer_bytes_total - - buffer-available-bytes: - metric_type: gauge - alias: kafka.producer.available_buffer_bytes - - bufferpool-wait-time: - metric_type: gauge - alias: kafka.producer.bufferpool_wait_time - - batch-size-avg: - metric_type: gauge - alias: kafka.producer.batch_size_avg - - batch-size-max: - metric_type: gauge - alias: kafka.producer.batch_size_max - - compression-rate-avg: - metric_type: gauge - alias: kafka.producer.compression_rate_avg - - record-queue-time-avg: - metric_type: gauge - alias: kafka.producer.record_queue_time_avg - - record-queue-time-max: - metric_type: gauge - alias: kafka.producer.record_queue_time_max - - request-latency-avg: - metric_type: gauge - alias: kafka.producer.request_latency_avg - - request-latency-max: - metric_type: gauge - alias: kafka.producer.request_latency_max - - record-send-rate: - metric_type: gauge - alias: kafka.producer.records_send_rate - - records-per-request-avg: - metric_type: gauge - alias: kafka.producer.records_per_request - - record-retry-rate: - metric_type: gauge - alias: kafka.producer.record_retry_rate - - record-error-rate: - metric_type: gauge - alias: kafka.producer.record_error_rate - - record-size-max: - metric_type: gauge - alias: kafka.producer.record_size_max - - record-size-avg: - metric_type: gauge - alias: kafka.producer.record_size_avg - - requests-in-flight: - metric_type: gauge - alias: kafka.producer.requests_in_flight - - metadata-age: - metric_type: gauge - alias: kafka.producer.metadata_age - - produce-throttle-time-max: - metric_type: gauge - alias: kafka.producer.throttle_time_max - - produce-throttle-time-avg: - metric_type: gauge - alias: kafka.producer.throttle_time_avg - - - # - # Producers: Per Topic Metrics - # - - include: - domain: 'kafka.producer' - bean_regex: 'kafka\.producer:type=producer-topic-metrics,client-id=(.*?),topic=(.*?)(?:,|$)' - attribute: - byte-rate: - metric_type: gauge - alias: kafka.producer.bytes_out - tags: - client: $1 - topic: $2 - - include: - domain: 'kafka.producer' - bean_regex: 'kafka\.producer:type=producer-topic-metrics,client-id=(.*?),topic=(.*?)(?:,|$)' - attribute: - record-send-rate: - metric_type: gauge - alias: kafka.producer.record_send_rate - tags: - client: $1 - topic: $2 - - include: - domain: 'kafka.producer' - bean_regex: 'kafka\.producer:type=producer-topic-metrics,client-id=(.*?),topic=(.*?)(?:,|$)' - attribute: - compression-rate: - metric_type: gauge - alias: kafka.producer.compression_rate - tags: - client: $1 - topic: $2 - - include: - domain: 'kafka.producer' - bean_regex: 'kafka\.producer:type=producer-topic-metrics,client-id=(.*?),topic=(.*?)(?:,|$)' - attribute: - record-retry-rate: - metric_type: gauge - alias: kafka.producer.record_retry_rate - tags: - client: $1 - topic: $2 - - include: - domain: 'kafka.producer' - bean_regex: 'kafka\.producer:type=producer-topic-metrics,client-id=(.*?),topic=(.*?)(?:,|$)' - attribute: - record-error-rate: - metric_type: gauge - alias: kafka.producer.record_error_rate - tags: - client: $1 - topic: $2 - - # - # Consumers (only v0.8.2.x) - # - - include: - domain: 'kafka.consumer' - bean_regex: 'kafka\.consumer:type=ConsumerFetcherManager,name=MaxLag,clientId=.*' - attribute: - Value: - metric_type: gauge - alias: kafka.consumer.max_lag - - include: - domain: 'kafka.consumer' - bean_regex: 'kafka\.consumer:type=ConsumerFetcherManager,name=MinFetchRate,clientId=.*' - attribute: - Value: - metric_type: gauge - alias: kafka.consumer.fetch_rate - - include: - domain: 'kafka.consumer' - bean_regex: 'kafka\.consumer:type=ConsumerTopicMetrics,name=BytesPerSec,clientId=.*' - attribute: - Count: - metric_type: rate - alias: kafka.consumer.bytes_in - - include: - domain: 'kafka.consumer' - bean_regex: 'kafka\.consumer:type=ConsumerTopicMetrics,name=MessagesPerSec,clientId=.*' - attribute: - Count: - metric_type: rate - alias: kafka.consumer.messages_in - - include: - # Offsets committed to ZooKeeper - domain: 'kafka.consumer' - bean_regex: 'kafka\.consumer:type=ZookeeperConsumerConnector,name=ZooKeeperCommitsPerSec,clientId=.*' - attribute: - Count: - metric_type: rate - alias: kafka.consumer.zookeeper_commits - - include: - # Offsets committed to Kafka - domain: 'kafka.consumer' - bean_regex: 'kafka\.consumer:type=ZookeeperConsumerConnector,name=KafkaCommitsPerSec,clientId=.*' - attribute: - Count: - metric_type: rate - alias: kafka.consumer.kafka_commits - - # - # Consumers (v0.9.0.x to v0.10.2.x) - # - - include: - domain: 'kafka.consumer' - bean_regex: 'kafka\.consumer:type=consumer-fetch-manager-metrics,client-id=.*' - attribute: - bytes-consumed-rate: - metric_type: gauge - alias: kafka.consumer.bytes_in - - include: - domain: 'kafka.consumer' - bean_regex: 'kafka\.consumer:type=consumer-fetch-manager-metrics,client-id=.*' - attribute: - records-consumed-rate: - metric_type: gauge - alias: kafka.consumer.messages_in - - # - # Consumers: Per Topic Metrics - # - - - include: - domain: 'kafka.consumer' - bean_regex: 'kafka\.consumer:type=consumer-fetch-manager-metrics,client-id=(.*?),topic=(.*?)(?:,|$)' - attribute: - fetch-size-avg: - metric_type: gauge - alias: kafka.consumer.fetch_size_avg - tags: - client: $1 - topic: $2 - - include: - domain: 'kafka.consumer' - bean_regex: 'kafka\.consumer:type=consumer-fetch-manager-metrics,client-id=(.*?),topic=(.*?)(?:,|$)' - attribute: - fetch-size-max: - metric_type: gauge - alias: kafka.consumer.fetch_size_max - tags: - client: $1 - topic: $2 - - include: - domain: 'kafka.consumer' - bean_regex: 'kafka\.consumer:type=consumer-fetch-manager-metrics,client-id=(.*?),topic=(.*?)(?:,|$)' - attribute: - bytes-consumed-rate: - metric_type: gauge - alias: kafka.consumer.bytes_consumed - tags: - client: $1 - topic: $2 - - include: - domain: 'kafka.consumer' - bean_regex: 'kafka\.consumer:type=consumer-fetch-manager-metrics,client-id=(.*?),topic=(.*?)(?:,|$)' - attribute: - records-per-request-avg: - metric_type: gauge - alias: kafka.consumer.records_per_request_avg - tags: - client: $1 - topic: $2 - - include: - domain: 'kafka.consumer' - bean_regex: 'kafka\.consumer:type=consumer-fetch-manager-metrics,client-id=(.*?),topic=(.*?)(?:,|$)' - attribute: - records-consumed-rate: - metric_type: gauge - alias: kafka.consumer.records_consumed - tags: - client: $1 - topic: $2 - - # - # Aggregate cluster stats - # - - include: - domain: 'kafka.server' - bean: 'kafka.server:type=BrokerTopicMetrics,name=BytesOutPerSec' - attribute: - Count: - metric_type: rate - alias: kafka.net.bytes_out.rate - - include: - domain: 'kafka.server' - bean: 'kafka.server:type=BrokerTopicMetrics,name=BytesInPerSec' - attribute: - Count: - metric_type: rate - alias: kafka.net.bytes_in.rate - - include: - domain: 'kafka.server' - bean: 'kafka.server:type=BrokerTopicMetrics,name=MessagesInPerSec' - attribute: - Count: - metric_type: rate - alias: kafka.messages_in.rate - - include: - domain: 'kafka.server' - bean: 'kafka.server:type=BrokerTopicMetrics,name=BytesRejectedPerSec' - attribute: - Count: - metric_type: rate - alias: kafka.net.bytes_rejected.rate - - # - # Per Topic Broker Stats - # - - include: - domain: '"kafka.server"' - bean_regex: '"kafka.server":type="BrokerTopicMetrics",name="(.*?)-MessagesInPerSec"' - attribute: - Count: - metric_type: rate - alias: kafka.topic.messages_in.rate - tags: - topic: $1 - - # - # Request timings - # - - include: - domain: 'kafka.server' - bean: 'kafka.server:type=BrokerTopicMetrics,name=FailedFetchRequestsPerSec' - attribute: - Count: - metric_type: rate - alias: kafka.request.fetch.failed.rate - - include: - domain: 'kafka.server' - bean: 'kafka.server:type=BrokerTopicMetrics,name=FailedProduceRequestsPerSec' - attribute: - Count: - metric_type: rate - alias: kafka.request.produce.failed.rate - - include: - domain: 'kafka.network' - bean: 'kafka.network:type=RequestMetrics,name=RequestsPerSec,request=Produce' - attribute: - Count: - metric_type: rate - alias: kafka.request.produce.rate - - include: - domain: 'kafka.network' - bean: 'kafka.network:type=RequestMetrics,name=TotalTimeMs,request=Produce' - attribute: - Mean: - metric_type: gauge - alias: kafka.request.produce.time.avg - 99thPercentile: - metric_type: gauge - alias: kafka.request.produce.time.99percentile - - include: - domain: 'kafka.network' - bean: 'kafka.network:type=RequestMetrics,name=RequestsPerSec,request=FetchConsumer' - attribute: - Count: - metric_type: rate - alias: kafka.request.fetch_consumer.rate - - include: - domain: 'kafka.network' - bean: 'kafka.network:type=RequestMetrics,name=RequestsPerSec,request=FetchFollower' - attribute: - Count: - metric_type: rate - alias: kafka.request.fetch_follower.rate - - include: - domain: 'kafka.network' - bean: 'kafka.network:type=RequestMetrics,name=TotalTimeMs,request=FetchConsumer' - attribute: - Mean: - metric_type: gauge - alias: kafka.request.fetch_consumer.time.avg - 99thPercentile: - metric_type: gauge - alias: kafka.request.fetch_consumer.time.99percentile - - include: - domain: 'kafka.network' - bean: 'kafka.network:type=RequestMetrics,name=TotalTimeMs,request=FetchFollower' - attribute: - Mean: - metric_type: gauge - alias: kafka.request.fetch_follower.time.avg - 99thPercentile: - metric_type: gauge - alias: kafka.request.fetch_follower.time.99percentile - - include: - domain: 'kafka.network' - bean: 'kafka.network:type=RequestMetrics,name=TotalTimeMs,request=UpdateMetadata' - attribute: - Mean: - metric_type: gauge - alias: kafka.request.update_metadata.time.avg - 99thPercentile: - metric_type: gauge - alias: kafka.request.update_metadata.time.99percentile - - include: - domain: 'kafka.network' - bean: 'kafka.network:type=RequestMetrics,name=TotalTimeMs,request=Metadata' - attribute: - Mean: - metric_type: gauge - alias: kafka.request.metadata.time.avg - 99thPercentile: - metric_type: gauge - alias: kafka.request.metadata.time.99percentile - - include: - domain: 'kafka.network' - bean: 'kafka.network:type=RequestMetrics,name=TotalTimeMs,request=Offsets' - attribute: - Mean: - metric_type: gauge - alias: kafka.request.offsets.time.avg - 99thPercentile: - metric_type: gauge - alias: kafka.request.offsets.time.99percentile - - include: - domain: 'kafka.server' - bean: 'kafka.server:type=KafkaRequestHandlerPool,name=RequestHandlerAvgIdlePercent' - attribute: - OneMinuteRate: - metric_type: gauge - alias: kafka.request.handler.avg.idle.pct.rate - - include: - domain: 'kafka.server' - bean: 'kafka.server:type=ProducerRequestPurgatory,name=PurgatorySize' - attribute: - Value: - metric_type: gauge - alias: kafka.request.producer_request_purgatory.size - - include: - domain: 'kafka.server' - bean: 'kafka.server:type=FetchRequestPurgatory,name=PurgatorySize' - attribute: - Value: - metric_type: gauge - alias: kafka.request.fetch_request_purgatory.size - - # - # Replication stats - # - - include: - domain: 'kafka.server' - bean: 'kafka.server:type=ReplicaManager,name=UnderReplicatedPartitions' - attribute: - Value: - metric_type: gauge - alias: kafka.replication.under_replicated_partitions - - include: - domain: 'kafka.server' - bean: 'kafka.server:type=ReplicaManager,name=IsrShrinksPerSec' - attribute: - Count: - metric_type: rate - alias: kafka.replication.isr_shrinks.rate - - include: - domain: 'kafka.server' - bean: 'kafka.server:type=ReplicaManager,name=IsrExpandsPerSec' - attribute: - Count: - metric_type: rate - alias: kafka.replication.isr_expands.rate - - include: - domain: 'kafka.controller' - bean: 'kafka.controller:type=ControllerStats,name=LeaderElectionRateAndTimeMs' - attribute: - Count: - metric_type: rate - alias: kafka.replication.leader_elections.rate - - include: - domain: 'kafka.controller' - bean: 'kafka.controller:type=ControllerStats,name=UncleanLeaderElectionsPerSec' - attribute: - Count: - metric_type: rate - alias: kafka.replication.unclean_leader_elections.rate - - include: - domain: 'kafka.controller' - bean: 'kafka.controller:type=KafkaController,name=OfflinePartitionsCount' - attribute: - Value: - metric_type: gauge - alias: kafka.replication.offline_partitions_count - - include: - domain: 'kafka.controller' - bean: 'kafka.controller:type=KafkaController,name=ActiveControllerCount' - attribute: - Value: - metric_type: gauge - alias: kafka.replication.active_controller_count - - include: - domain: 'kafka.server' - bean: 'kafka.server:type=ReplicaManager,name=PartitionCount' - attribute: - Value: - metric_type: gauge - alias: kafka.replication.partition_count - - include: - domain: 'kafka.server' - bean: 'kafka.server:type=ReplicaManager,name=LeaderCount' - attribute: - Value: - metric_type: gauge - alias: kafka.replication.leader_count - - include: - domain: 'kafka.server' - bean: 'kafka.server:type=ReplicaFetcherManager,name=MaxLag,clientId=Replica' - attribute: - Value: - metric_type: gauge - alias: kafka.replication.max_lag - - # - # Log flush stats - # - - include: - domain: 'kafka.log' - bean: 'kafka.log:type=LogFlushStats,name=LogFlushRateAndTimeMs' - attribute: - Count: - metric_type: rate - alias: kafka.log.flush_rate.rate diff --git a/dd-java-agent/agent-jmxfetch/src/main/resources/datadog/trace/agent/jmxfetch/metricconfigs/solr.yaml b/dd-java-agent/agent-jmxfetch/src/main/resources/datadog/trace/agent/jmxfetch/metricconfigs/solr.yaml deleted file mode 100644 index 3fa966c5a0..0000000000 --- a/dd-java-agent/agent-jmxfetch/src/main/resources/datadog/trace/agent/jmxfetch/metricconfigs/solr.yaml +++ /dev/null @@ -1,200 +0,0 @@ -# Default metrics collected by this check. You should not have to modify this. -jmx_metrics: - - # Solr versions >= 7.0 - - include: - category: SEARCHER - name: - - numDocs - scope: - - searcher - attribute: - Value: - alias: solr.searcher.numdocs - metric_type: gauge - - - include: - category: SEARCHER - name: maxDoc - scope: - - searcher - attribute: - Value: - alias: solr.searcher.maxdocs - metric_type: gauge - - - include: - category: SEARCHER - name: warmupTime - scope: - - searcher - attribute: - Value: - alias: solr.searcher.warmup - metric_type: gauge - - - include: - category: CACHE - name: documentCache - scope: - - searcher - attribute: - cumulative_lookups: - alias: solr.document_cache.lookups - metric_type: counter - cumulative_hits: - alias: solr.document_cache.hits - metric_type: counter - cumulative_inserts: - alias: solr.document_cache.inserts - metric_type: counter - cumulative_evictions: - alias: solr.document_cache.evictions - metric_type: counter - - - include: - category: CACHE - name: queryResultCache - scope: - - searcher - attribute: - cumulative_lookups: - alias: solr.query_result_cache.lookups - metric_type: counter - cumulative_hits: - alias: solr.query_result_cache.hits - metric_type: counter - cumulative_inserts: - alias: solr.query_result_cache.inserts - metric_type: counter - cumulative_evictions: - alias: solr.query_result_cache.evictions - metric_type: counter - - - include: - category: CACHE - name: filterCache - scope: - - searcher - attribute: - cumulative_lookups: - alias: solr.filter_cache.lookups - metric_type: counter - cumulative_hits: - alias: solr.filter_cache.hits - metric_type: counter - cumulative_inserts: - alias: solr.filter_cache.inserts - metric_type: counter - cumulative_evictions: - alias: solr.filter_cache.evictions - metric_type: counter - - - include: - category: QUERY - name: requests - attribute: - Count: - alias: solr.search_handler.requests - metric_type: counter - - - include: - category: QUERY - name: timeouts - attribute: - Count: - alias: solr.search_handler.timeouts - metric_type: counter - - - include: - category: QUERY - name: errors - attribute: - Count: - alias: solr.search_handler.errors - metric_type: counter - - - include: - category: QUERY - name: totalTime - attribute: - Count: - alias: solr.search_handler.time - metric_type: counter - - - include: - category: QUERY - name: requestTimes - attribute: - avgRequestsPerSecond: - alias: solr.search_handler.avg_requests_per_sec - metric_type: gauge - avgTimePerRequest: - alias: solr.search_handler.avg_time_per_req - metric_type: gauge - - - # Solr versions < 7.0 - - include: - type: searcher - attribute: - maxDoc: - alias: solr.searcher.maxdoc - metric_type: gauge - numDocs: - alias: solr.searcher.numdocs - metric_type: gauge - warmupTime: - alias: solr.searcher.warmup - metric_type: gauge - - include: - id: org.apache.solr.search.FastLRUCache - attribute: - cumulative_lookups: - alias: solr.cache.lookups - metric_type: counter - cumulative_hits: - alias: solr.cache.hits - metric_type: counter - cumulative_inserts: - alias: solr.cache.inserts - metric_type: counter - cumulative_evictions: - alias: solr.cache.evictions - metric_type: counter - - include: - id: org.apache.solr.search.LRUCache - attribute: - cumulative_lookups: - alias: solr.cache.lookups - metric_type: counter - cumulative_hits: - alias: solr.cache.hits - metric_type: counter - cumulative_inserts: - alias: solr.cache.inserts - metric_type: counter - cumulative_evictions: - alias: solr.cache.evictions - metric_type: counter - - include: - id: org.apache.solr.handler.component.SearchHandler - attribute: - errors: - alias: solr.search_handler.errors - metric_type: counter - requests: - alias: solr.search_handler.requests - metric_type: counter - timeouts: - alias: solr.search_handler.timeouts - metric_type: counter - totalTime: - alias: solr.search_handler.time - metric_type: counter - avgTimePerRequest: - alias: solr.search_handler.avg_time_per_req - metric_type: gauge - avgRequestsPerSecond: - alias: solr.search_handler.avg_requests_per_sec - metric_type: gauge diff --git a/dd-java-agent/agent-jmxfetch/src/main/resources/datadog/trace/agent/jmxfetch/metricconfigs/tomcat.yaml b/dd-java-agent/agent-jmxfetch/src/main/resources/datadog/trace/agent/jmxfetch/metricconfigs/tomcat.yaml deleted file mode 100644 index 370cb90421..0000000000 --- a/dd-java-agent/agent-jmxfetch/src/main/resources/datadog/trace/agent/jmxfetch/metricconfigs/tomcat.yaml +++ /dev/null @@ -1,65 +0,0 @@ -# Default metrics collected by this check. You should not have to modify this. -jmx_metrics: - - include: - type: ThreadPool - attribute: - maxThreads: - alias: tomcat.threads.max - metric_type: gauge - currentThreadCount: - alias: tomcat.threads.count - metric_type: gauge - currentThreadsBusy: - alias: tomcat.threads.busy - metric_type: gauge - - include: - type: GlobalRequestProcessor - attribute: - bytesSent: - alias: tomcat.bytes_sent - metric_type: counter - bytesReceived: - alias: tomcat.bytes_rcvd - metric_type: counter - errorCount: - alias: tomcat.error_count - metric_type: counter - requestCount: - alias: tomcat.request_count - metric_type: counter - maxTime: - alias: tomcat.max_time - metric_type: gauge - processingTime: - alias: tomcat.processing_time - metric_type: counter - - include: - j2eeType: Servlet - attribute: - processingTime: - alias: tomcat.servlet.processing_time - metric_type: counter - errorCount: - alias: tomcat.servlet.error_count - metric_type: counter - requestCount: - alias: tomcat.servlet.request_count - metric_type: counter - - include: - type: Cache - attribute: - accessCount: - alias: tomcat.cache.access_count - metric_type: counter - hitsCounts: - alias: tomcat.cache.hits_count - metric_type: counter - - include: - type: JspMonitor - attribute: - jspCount: - alias: tomcat.jsp.count - metric_type: counter - jspReloadCount: - alias: tomcat.jsp.reload_count - metric_type: counter From 1df20cf52cbd6275275e4f9b01f6e650897a6442 Mon Sep 17 00:00:00 2001 From: Tyler Benson Date: Wed, 23 Jan 2019 13:10:27 -0800 Subject: [PATCH 4/4] Upgrade to latest version of jmxfetch This includes the ability to load config from resources. --- dd-java-agent/agent-jmxfetch/agent-jmxfetch.gradle | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/dd-java-agent/agent-jmxfetch/agent-jmxfetch.gradle b/dd-java-agent/agent-jmxfetch/agent-jmxfetch.gradle index 5bb89f4109..fdfa813f64 100644 --- a/dd-java-agent/agent-jmxfetch/agent-jmxfetch.gradle +++ b/dd-java-agent/agent-jmxfetch/agent-jmxfetch.gradle @@ -4,7 +4,7 @@ plugins { apply from: "${rootDir}/gradle/java.gradle" dependencies { - compile 'com.datadoghq:jmxfetch:0.24.0' + compile 'com.datadoghq:jmxfetch:0.25.0' compile deps.slf4j compile project(':dd-trace-api') } @@ -38,6 +38,10 @@ task copyMetricConfigs(type:Exec) { group 'Build Setup' description 'Copy metrics.yaml files from integrations-core into resources' commandLine './copy-metric-configs.sh', 'integrations-core', sourceSets.main.output.resourcesDir + doFirst { + // Ensure the resources directory is available. + file(sourceSets.main.output.resourcesDir).mkdirs() + } } processResources.dependsOn copyMetricConfigs