From de166e5ecb994072ceb403e4aabc4a708fcedce8 Mon Sep 17 00:00:00 2001 From: Tyler Benson Date: Wed, 2 Jan 2019 10:47:28 -0800 Subject: [PATCH] First pass at embedding existing jmx integrations Using copy/paste from integrations-core. Will depend on a new jmxfetch release with https://github.com/DataDog/jmxfetch/pull/205 before this will work. --- .../trace/agent/jmxfetch/JMXFetch.java | 35 +- .../trace/agent/jmxfetch/metricconfigs.txt | 5 + .../jmxfetch/metricconfigs/activemq.yaml | 54 ++ .../jmxfetch/metricconfigs/cassandra.yaml | 427 +++++++++++++ .../agent/jmxfetch/metricconfigs/kafka.yaml | 566 ++++++++++++++++++ .../agent/jmxfetch/metricconfigs/solr.yaml | 200 +++++++ .../agent/jmxfetch/metricconfigs/tomcat.yaml | 65 ++ 7 files changed, 1351 insertions(+), 1 deletion(-) create mode 100644 dd-java-agent/agent-jmxfetch/src/main/resources/datadog/trace/agent/jmxfetch/metricconfigs.txt create mode 100644 dd-java-agent/agent-jmxfetch/src/main/resources/datadog/trace/agent/jmxfetch/metricconfigs/activemq.yaml create mode 100644 dd-java-agent/agent-jmxfetch/src/main/resources/datadog/trace/agent/jmxfetch/metricconfigs/cassandra.yaml create mode 100644 dd-java-agent/agent-jmxfetch/src/main/resources/datadog/trace/agent/jmxfetch/metricconfigs/kafka.yaml create mode 100644 dd-java-agent/agent-jmxfetch/src/main/resources/datadog/trace/agent/jmxfetch/metricconfigs/solr.yaml create mode 100644 dd-java-agent/agent-jmxfetch/src/main/resources/datadog/trace/agent/jmxfetch/metricconfigs/tomcat.yaml diff --git a/dd-java-agent/agent-jmxfetch/src/main/java/datadog/trace/agent/jmxfetch/JMXFetch.java b/dd-java-agent/agent-jmxfetch/src/main/java/datadog/trace/agent/jmxfetch/JMXFetch.java index 99770f6c75..83d28550f3 100644 --- a/dd-java-agent/agent-jmxfetch/src/main/java/datadog/trace/agent/jmxfetch/JMXFetch.java +++ b/dd-java-agent/agent-jmxfetch/src/main/java/datadog/trace/agent/jmxfetch/JMXFetch.java @@ -2,9 +2,16 @@ package datadog.trace.agent.jmxfetch; import com.google.common.collect.ImmutableList; import datadog.trace.api.Config; +import java.io.IOException; +import java.io.InputStream; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Map; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.io.IOUtils; import org.datadog.jmxfetch.App; import org.datadog.jmxfetch.AppConfig; @@ -27,6 +34,7 @@ public class JMXFetch { return; } + final List internalMetricsConfigs = getInternalMetricFiles(); final List metricsConfigs = config.getJmxFetchMetricsConfigs(); final Integer checkPeriod = config.getJmxFetchCheckPeriod(); final Integer refreshBeansPeriod = config.getJmxFetchRefreshBeansPeriod(); @@ -36,7 +44,8 @@ public class JMXFetch { final String logLevel = getLogLevel(); log.error( - "JMXFetch config: {} {} {} {} {} {} {}", + "JMXFetch config: {} {} {} {} {} {} {} {}", + internalMetricsConfigs, metricsConfigs, checkPeriod, refreshBeansPeriod, @@ -47,6 +56,7 @@ public class JMXFetch { final AppConfig appConfig = AppConfig.create( DEFAULT_CONFIGS, + internalMetricsConfigs, metricsConfigs, checkPeriod, refreshBeansPeriod, @@ -95,6 +105,29 @@ public class JMXFetch { return "statsd:" + host + ":" + config.getJmxFetchStatsdPort(); } + private static List getInternalMetricFiles() { + try { + final InputStream metricConfigsStream = + JMXFetch.class.getResourceAsStream("metricconfigs.txt"); + if (metricConfigsStream == null) { + log.debug("metricconfigs not found. returning empty set"); + return Collections.emptyList(); + } else { + final String configs = IOUtils.toString(metricConfigsStream, StandardCharsets.UTF_8); + final String[] split = configs.split("\n"); + final List result = new ArrayList<>(split.length); + for (final String config : split) { + final URL resource = JMXFetch.class.getResource("metricconfigs/" + config); + result.add(resource.getPath().split("\\.jar!/")[1]); + } + return result; + } + } catch (final IOException e) { + log.debug("error reading metricconfigs. returning empty set", e); + return Collections.emptyList(); + } + } + private static String getLogLocation() { return System.getProperty("org.slf4j.simpleLogger.logFile", "System.err"); } diff --git a/dd-java-agent/agent-jmxfetch/src/main/resources/datadog/trace/agent/jmxfetch/metricconfigs.txt b/dd-java-agent/agent-jmxfetch/src/main/resources/datadog/trace/agent/jmxfetch/metricconfigs.txt new file mode 100644 index 0000000000..521c7d2204 --- /dev/null +++ b/dd-java-agent/agent-jmxfetch/src/main/resources/datadog/trace/agent/jmxfetch/metricconfigs.txt @@ -0,0 +1,5 @@ +activemq.yaml +cassandra.yaml +kafka.yaml +solr.yaml +tomcat.yaml diff --git a/dd-java-agent/agent-jmxfetch/src/main/resources/datadog/trace/agent/jmxfetch/metricconfigs/activemq.yaml b/dd-java-agent/agent-jmxfetch/src/main/resources/datadog/trace/agent/jmxfetch/metricconfigs/activemq.yaml new file mode 100644 index 0000000000..d0735950ba --- /dev/null +++ b/dd-java-agent/agent-jmxfetch/src/main/resources/datadog/trace/agent/jmxfetch/metricconfigs/activemq.yaml @@ -0,0 +1,54 @@ +# Default metrics collected by this check. You should not have to modify this. +jmx_metrics: +- include: + destinationType: Queue + attribute: + AverageEnqueueTime: + alias: activemq.queue.avg_enqueue_time + metric_type: gauge + ConsumerCount: + alias: activemq.queue.consumer_count + metric_type: gauge + ProducerCount: + alias: activemq.queue.producer_count + metric_type: gauge + MaxEnqueueTime: + alias: activemq.queue.max_enqueue_time + metric_type: gauge + MinEnqueueTime: + alias: activemq.queue.min_enqueue_time + metric_type: gauge + MemoryPercentUsage: + alias: activemq.queue.memory_pct + metric_type: gauge + QueueSize: + alias: activemq.queue.size + metric_type: gauge + DequeueCount: + alias: activemq.queue.dequeue_count + metric_type: counter + DispatchCount: + alias: activemq.queue.dispatch_count + metric_type: counter + EnqueueCount: + alias: activemq.queue.enqueue_count + metric_type: counter + ExpiredCount: + alias: activemq.queue.expired_count + metric_type: counter + InFlightCount: + alias: activemq.queue.in_flight_count + metric_type: counter + +- include: + type: Broker + attribute: + StorePercentUsage: + alias: activemq.broker.store_pct + metric_type: gauge + TempPercentUsage: + alias: activemq.broker.temp_pct + metric_type: gauge + MemoryPercentUsage: + alias: activemq.broker.memory_pct + metric_type: gauge diff --git a/dd-java-agent/agent-jmxfetch/src/main/resources/datadog/trace/agent/jmxfetch/metricconfigs/cassandra.yaml b/dd-java-agent/agent-jmxfetch/src/main/resources/datadog/trace/agent/jmxfetch/metricconfigs/cassandra.yaml new file mode 100644 index 0000000000..a94c31374d --- /dev/null +++ b/dd-java-agent/agent-jmxfetch/src/main/resources/datadog/trace/agent/jmxfetch/metricconfigs/cassandra.yaml @@ -0,0 +1,427 @@ +# Default metrics collected by this check. You should not have to modify this. +jmx_metrics: + - include: + domain: org.apache.cassandra.metrics + type: ClientRequest + name: + - Latency + attribute: + - 75thPercentile + - 95thPercentile + - OneMinuteRate + - include: + domain: org.apache.cassandra.metrics + type: DroppedMessage + name: + - Dropped + attribute: + - OneMinuteRate + - include: + domain: org.apache.cassandra.metrics + type: ThreadPools + scope: + - MutationStage + - CounterMutationStage + - ReadStage + - ViewMutationStage + name: + - PendingTasks + - CurrentlyBlockedTasks + path: + - request + - include: + domain: org.apache.cassandra.metrics + type: ThreadPools + scope: + - MemtableFlushWriter + - HintsDispatcher + - MemtablePostFlush + - MigrationStage + - MiscStage + - SecondaryIndexManagement + name: + - PendingTasks + - CurrentlyBlockedTasks + path: + - internal + - include: + domain: org.apache.cassandra.metrics + type: Storage + name: + - Load + - Exceptions + - include: + domain: org.apache.cassandra.metrics + type: Table + bean_regex: + - .*keyspace=.* + name: + - ReadLatency + - WriteLatency + attribute: + - 75thPercentile + - 95thPercentile + - 99thPercentile + - OneMinuteRate + exclude: + keyspace: + - system + - system_auth + - system_distributed + - system_schema + - system_traces + - include: + domain: org.apache.cassandra.metrics + type: Table + bean_regex: + - .*keyspace=.* + name: + - RangeLatency + - CasPrepareLatency + - CasProposeLatency + - CasCommitLatency + - ViewLockAcquireTime + - ViewReadTime + attribute: + - 75thPercentile + - 95thPercentile + - OneMinuteRate + exclude: + keyspace: + - system + - system_auth + - system_distributed + - system_schema + - system_traces + - include: + domain: org.apache.cassandra.metrics + type: Table + bean_regex: + - .*keyspace=.* + name: + - SSTablesPerReadHistogram + - TombstoneScannedHistogram + - WaitingOnFreeMemtableSpace + attribute: + - 75thPercentile + - 95thPercentile + exclude: + keyspace: + - system + - system_auth + - system_distributed + - system_schema + - system_traces + - include: + domain: org.apache.cassandra.metrics + type: Table + bean_regex: + - .*keyspace=.* + name: + - ColUpdateTimeDeltaHistogram + attribute: + - Min + - 75thPercentile + - 95thPercentile + exclude: + keyspace: + - system + - system_auth + - system_distributed + - system_schema + - system_traces + - include: + domain: org.apache.cassandra.metrics + type: Table + bean_regex: + - .*keyspace=.* + name: + - BloomFilterFalseRatio + - CompressionRatio + - KeyCacheHitRate + - LiveSSTableCount + - MaxPartitionSize + - MeanPartitionSize + - MeanRowSize + - MaxRowSize + - PendingCompactions + - SnapshotsSize + attribute: + - Value + exclude: + keyspace: + - system + - system_auth + - system_distributed + - system_schema + - system_traces + - include: + domain: org.apache.cassandra.metrics + type: Table + bean_regex: + - .*keyspace=.* + name: + - CompactionBytesWritten + - BytesFlushed + - PendingFlushes + - LiveDiskSpaceUsed + - TotalDiskSpaceUsed + - RowCacheHitOutOfRange + - RowCacheHit + - RowCacheMiss + attribute: + - Count + exclude: + keyspace: + - system + - system_auth + - system_distributed + - system_schema + - system_traces + - include: + domain: org.apache.cassandra.metrics + type: Cache + scope: KeyCache + name: + - HitRate + attribute: + - Count + - include: + domain: org.apache.cassandra.metrics + type: CommitLog + name: + - PendingTasks + - TotalCommitLogSize + attribute: + - Value + - include: + domain: org.apache.cassandra.db + type: Tables + attribute: + DroppableTombstoneRatio: + alias: cassandra.db.droppable_tombstone_ratio + # Young Gen Collectors (Minor Collections) + - include: + domain: java.lang + type: GarbageCollector + name: Copy + attribute: + CollectionCount: + metric_type: counter + alias: jmx.gc.minor_collection_count + CollectionTime: + metric_type: counter + alias: jmx.gc.minor_collection_time + - include: + domain: java.lang + type: GarbageCollector + name: PS Scavenge + attribute: + CollectionCount: + metric_type: counter + alias: jmx.gc.minor_collection_count + CollectionTime: + metric_type: counter + alias: jmx.gc.minor_collection_time + - include: + domain: java.lang + type: GarbageCollector + name: ParNew + attribute: + CollectionCount: + metric_type: counter + alias: jmx.gc.minor_collection_count + CollectionTime: + metric_type: counter + alias: jmx.gc.minor_collection_time + - include: + domain: java.lang + type: GarbageCollector + name: G1 Young Generation + attribute: + CollectionCount: + metric_type: counter + alias: jmx.gc.minor_collection_count + CollectionTime: + metric_type: counter + alias: jmx.gc.minor_collection_time + # Old Gen Collectors (Major collections) + - include: + domain: java.lang + type: GarbageCollector + name: MarkSweepCompact + attribute: + CollectionCount: + metric_type: counter + alias: jmx.gc.major_collection_count + CollectionTime: + metric_type: counter + alias: jmx.gc.major_collection_time + - include: + domain: java.lang + type: GarbageCollector + name: PS MarkSweep + attribute: + CollectionCount: + metric_type: counter + alias: jmx.gc.major_collection_count + CollectionTime: + metric_type: counter + alias: jmx.gc.major_collection_time + - include: + domain: java.lang + type: GarbageCollector + name: ConcurrentMarkSweep + attribute: + CollectionCount: + metric_type: counter + alias: jmx.gc.major_collection_count + CollectionTime: + metric_type: counter + alias: jmx.gc.major_collection_time + - include: + domain: java.lang + type: GarbageCollector + name: G1 Mixed Generation + attribute: + CollectionCount: + metric_type: counter + alias: jmx.gc.major_collection_count + CollectionTime: + metric_type: counter + alias: jmx.gc.major_collection_time + # Deprecated metrics for pre Cassandra 3.0 versions compatibility. + # If you are using cassandra 2, the metrics below will be used, + # otherwise they will be ignored. + - include: + domain: org.apache.cassandra.metrics + type: ColumnFamily + bean_regex: + - .*keyspace=.* + name: + - ReadLatency + - WriteLatency + attribute: + - 75thPercentile + - 95thPercentile + - 99thPercentile + - OneMinuteRate + exclude: + keyspace: + - system + - system_auth + - system_distributed + - system_schema + - system_traces + - include: + domain: org.apache.cassandra.metrics + type: ColumnFamily + bean_regex: + - .*keyspace=.* + name: + - RangeLatency + - CasPrepareLatency + - CasProposeLatency + - CasCommitLatency + - ViewLockAcquireTime + - ViewReadTime + attribute: + - 75thPercentile + - 95thPercentile + - OneMinuteRate + exclude: + keyspace: + - system + - system_auth + - system_distributed + - system_schema + - system_traces + - include: + domain: org.apache.cassandra.metrics + type: ColumnFamily + bean_regex: + - .*keyspace=.* + name: + - SSTablesPerReadHistogram + - TombstoneScannedHistogram + - WaitingOnFreeMemtableSpace + attribute: + - 75thPercentile + - 95thPercentile + exclude: + keyspace: + - system + - system_auth + - system_distributed + - system_schema + - system_traces + - include: + domain: org.apache.cassandra.metrics + type: ColumnFamily + bean_regex: + - .*keyspace=.* + name: + - ColUpdateTimeDeltaHistogram + attribute: + - Min + - 75thPercentile + - 95thPercentile + exclude: + keyspace: + - system + - system_auth + - system_distributed + - system_schema + - system_traces + - include: + domain: org.apache.cassandra.metrics + type: ColumnFamily + bean_regex: + - .*keyspace=.* + name: + - BloomFilterFalseRatio + - CompressionRatio + - KeyCacheHitRate + - LiveSSTableCount + - MaxPartitionSize + - MeanPartitionSize + - MeanRowSize + - MaxRowSize + - PendingCompactions + - SnapshotsSize + attribute: + - Value + exclude: + keyspace: + - system + - system_auth + - system_distributed + - system_schema + - system_traces + - include: + domain: org.apache.cassandra.metrics + type: ColumnFamily + bean_regex: + - .*keyspace=.* + name: + - PendingFlushes + - LiveDiskSpaceUsed + - TotalDiskSpaceUsed + - RowCacheHitOutOfRange + - RowCacheHit + - RowCacheMiss + attribute: + - Count + exclude: + keyspace: + - system + - system_auth + - system_distributed + - system_schema + - system_traces + - include: + domain: org.apache.cassandra.db + type: ColumnFamilies + attribute: + DroppableTombstoneRatio: + alias: cassandra.db.droppable_tombstone_ratio diff --git a/dd-java-agent/agent-jmxfetch/src/main/resources/datadog/trace/agent/jmxfetch/metricconfigs/kafka.yaml b/dd-java-agent/agent-jmxfetch/src/main/resources/datadog/trace/agent/jmxfetch/metricconfigs/kafka.yaml new file mode 100644 index 0000000000..6bb326e570 --- /dev/null +++ b/dd-java-agent/agent-jmxfetch/src/main/resources/datadog/trace/agent/jmxfetch/metricconfigs/kafka.yaml @@ -0,0 +1,566 @@ +# Metrics collected by this check. You should not have to modify this. +jmx_metrics: + # + # Producers (only v0.8.2.x) + # + - include: + domain: 'kafka.producer' + bean_regex: 'kafka\.producer:type=ProducerRequestMetrics,name=ProducerRequestRateAndTimeMs,clientId=.*' + attribute: + Count: + metric_type: rate + alias: kafka.producer.request_rate + - include: + domain: 'kafka.producer' + bean_regex: 'kafka\.producer:type=ProducerRequestMetrics,name=ProducerRequestRateAndTimeMs,clientId=.*' + attribute: + Mean: + metric_type: gauge + alias: kafka.producer.request_latency_avg + - include: + domain: 'kafka.producer' + bean_regex: 'kafka\.producer:type=ProducerTopicMetrics,name=BytesPerSec,clientId=.*' + attribute: + Count: + metric_type: rate + alias: kafka.producer.bytes_out + - include: + domain: 'kafka.producer' + bean_regex: 'kafka\.producer:type=ProducerTopicMetrics,name=MessagesPerSec,clientId=.*' + attribute: + Count: + metric_type: rate + alias: kafka.producer.message_rate + + # + # Producers (v0.9.0.x to v0.10.2.x) + # + - include: + domain: 'kafka.producer' + bean_regex: 'kafka\.producer:type=producer-metrics,client-id=.*' + attribute: + response-rate: + metric_type: gauge + alias: kafka.producer.response_rate + - include: + domain: 'kafka.producer' + bean_regex: 'kafka\.producer:type=producer-metrics,client-id=.*' + attribute: + request-rate: + metric_type: gauge + alias: kafka.producer.request_rate + - include: + domain: 'kafka.producer' + bean_regex: 'kafka\.producer:type=producer-metrics,client-id=.*' + attribute: + request-latency-avg: + metric_type: gauge + alias: kafka.producer.request_latency_avg + - include: + domain: 'kafka.producer' + bean_regex: 'kafka\.producer:type=producer-metrics,client-id=.*' + attribute: + outgoing-byte-rate: + metric_type: gauge + alias: kafka.producer.bytes_out + - include: + domain: 'kafka.producer' + bean_regex: 'kafka\.producer:type=producer-metrics,client-id=.*' + attribute: + io-wait-time-ns-avg: + metric_type: gauge + alias: kafka.producer.io_wait + + - include: + domain: 'kafka.producer' + bean_regex: 'kafka.producer:type=producer-metrics,client-id=([-.\w]+)' + attribute: + - waiting-threads: + metric_type: gauge + alias: kafka.producer.waiting_threads + - buffer-total-bytes: + metric_type: gauge + alias: kafka.producer.buffer_bytes_total + - buffer-available-bytes: + metric_type: gauge + alias: kafka.producer.available_buffer_bytes + - bufferpool-wait-time: + metric_type: gauge + alias: kafka.producer.bufferpool_wait_time + - batch-size-avg: + metric_type: gauge + alias: kafka.producer.batch_size_avg + - batch-size-max: + metric_type: gauge + alias: kafka.producer.batch_size_max + - compression-rate-avg: + metric_type: gauge + alias: kafka.producer.compression_rate_avg + - record-queue-time-avg: + metric_type: gauge + alias: kafka.producer.record_queue_time_avg + - record-queue-time-max: + metric_type: gauge + alias: kafka.producer.record_queue_time_max + - request-latency-avg: + metric_type: gauge + alias: kafka.producer.request_latency_avg + - request-latency-max: + metric_type: gauge + alias: kafka.producer.request_latency_max + - record-send-rate: + metric_type: gauge + alias: kafka.producer.records_send_rate + - records-per-request-avg: + metric_type: gauge + alias: kafka.producer.records_per_request + - record-retry-rate: + metric_type: gauge + alias: kafka.producer.record_retry_rate + - record-error-rate: + metric_type: gauge + alias: kafka.producer.record_error_rate + - record-size-max: + metric_type: gauge + alias: kafka.producer.record_size_max + - record-size-avg: + metric_type: gauge + alias: kafka.producer.record_size_avg + - requests-in-flight: + metric_type: gauge + alias: kafka.producer.requests_in_flight + - metadata-age: + metric_type: gauge + alias: kafka.producer.metadata_age + - produce-throttle-time-max: + metric_type: gauge + alias: kafka.producer.throttle_time_max + - produce-throttle-time-avg: + metric_type: gauge + alias: kafka.producer.throttle_time_avg + + + # + # Producers: Per Topic Metrics + # + - include: + domain: 'kafka.producer' + bean_regex: 'kafka\.producer:type=producer-topic-metrics,client-id=(.*?),topic=(.*?)(?:,|$)' + attribute: + byte-rate: + metric_type: gauge + alias: kafka.producer.bytes_out + tags: + client: $1 + topic: $2 + - include: + domain: 'kafka.producer' + bean_regex: 'kafka\.producer:type=producer-topic-metrics,client-id=(.*?),topic=(.*?)(?:,|$)' + attribute: + record-send-rate: + metric_type: gauge + alias: kafka.producer.record_send_rate + tags: + client: $1 + topic: $2 + - include: + domain: 'kafka.producer' + bean_regex: 'kafka\.producer:type=producer-topic-metrics,client-id=(.*?),topic=(.*?)(?:,|$)' + attribute: + compression-rate: + metric_type: gauge + alias: kafka.producer.compression_rate + tags: + client: $1 + topic: $2 + - include: + domain: 'kafka.producer' + bean_regex: 'kafka\.producer:type=producer-topic-metrics,client-id=(.*?),topic=(.*?)(?:,|$)' + attribute: + record-retry-rate: + metric_type: gauge + alias: kafka.producer.record_retry_rate + tags: + client: $1 + topic: $2 + - include: + domain: 'kafka.producer' + bean_regex: 'kafka\.producer:type=producer-topic-metrics,client-id=(.*?),topic=(.*?)(?:,|$)' + attribute: + record-error-rate: + metric_type: gauge + alias: kafka.producer.record_error_rate + tags: + client: $1 + topic: $2 + + # + # Consumers (only v0.8.2.x) + # + - include: + domain: 'kafka.consumer' + bean_regex: 'kafka\.consumer:type=ConsumerFetcherManager,name=MaxLag,clientId=.*' + attribute: + Value: + metric_type: gauge + alias: kafka.consumer.max_lag + - include: + domain: 'kafka.consumer' + bean_regex: 'kafka\.consumer:type=ConsumerFetcherManager,name=MinFetchRate,clientId=.*' + attribute: + Value: + metric_type: gauge + alias: kafka.consumer.fetch_rate + - include: + domain: 'kafka.consumer' + bean_regex: 'kafka\.consumer:type=ConsumerTopicMetrics,name=BytesPerSec,clientId=.*' + attribute: + Count: + metric_type: rate + alias: kafka.consumer.bytes_in + - include: + domain: 'kafka.consumer' + bean_regex: 'kafka\.consumer:type=ConsumerTopicMetrics,name=MessagesPerSec,clientId=.*' + attribute: + Count: + metric_type: rate + alias: kafka.consumer.messages_in + - include: + # Offsets committed to ZooKeeper + domain: 'kafka.consumer' + bean_regex: 'kafka\.consumer:type=ZookeeperConsumerConnector,name=ZooKeeperCommitsPerSec,clientId=.*' + attribute: + Count: + metric_type: rate + alias: kafka.consumer.zookeeper_commits + - include: + # Offsets committed to Kafka + domain: 'kafka.consumer' + bean_regex: 'kafka\.consumer:type=ZookeeperConsumerConnector,name=KafkaCommitsPerSec,clientId=.*' + attribute: + Count: + metric_type: rate + alias: kafka.consumer.kafka_commits + + # + # Consumers (v0.9.0.x to v0.10.2.x) + # + - include: + domain: 'kafka.consumer' + bean_regex: 'kafka\.consumer:type=consumer-fetch-manager-metrics,client-id=.*' + attribute: + bytes-consumed-rate: + metric_type: gauge + alias: kafka.consumer.bytes_in + - include: + domain: 'kafka.consumer' + bean_regex: 'kafka\.consumer:type=consumer-fetch-manager-metrics,client-id=.*' + attribute: + records-consumed-rate: + metric_type: gauge + alias: kafka.consumer.messages_in + + # + # Consumers: Per Topic Metrics + # + + - include: + domain: 'kafka.consumer' + bean_regex: 'kafka\.consumer:type=consumer-fetch-manager-metrics,client-id=(.*?),topic=(.*?)(?:,|$)' + attribute: + fetch-size-avg: + metric_type: gauge + alias: kafka.consumer.fetch_size_avg + tags: + client: $1 + topic: $2 + - include: + domain: 'kafka.consumer' + bean_regex: 'kafka\.consumer:type=consumer-fetch-manager-metrics,client-id=(.*?),topic=(.*?)(?:,|$)' + attribute: + fetch-size-max: + metric_type: gauge + alias: kafka.consumer.fetch_size_max + tags: + client: $1 + topic: $2 + - include: + domain: 'kafka.consumer' + bean_regex: 'kafka\.consumer:type=consumer-fetch-manager-metrics,client-id=(.*?),topic=(.*?)(?:,|$)' + attribute: + bytes-consumed-rate: + metric_type: gauge + alias: kafka.consumer.bytes_consumed + tags: + client: $1 + topic: $2 + - include: + domain: 'kafka.consumer' + bean_regex: 'kafka\.consumer:type=consumer-fetch-manager-metrics,client-id=(.*?),topic=(.*?)(?:,|$)' + attribute: + records-per-request-avg: + metric_type: gauge + alias: kafka.consumer.records_per_request_avg + tags: + client: $1 + topic: $2 + - include: + domain: 'kafka.consumer' + bean_regex: 'kafka\.consumer:type=consumer-fetch-manager-metrics,client-id=(.*?),topic=(.*?)(?:,|$)' + attribute: + records-consumed-rate: + metric_type: gauge + alias: kafka.consumer.records_consumed + tags: + client: $1 + topic: $2 + + # + # Aggregate cluster stats + # + - include: + domain: 'kafka.server' + bean: 'kafka.server:type=BrokerTopicMetrics,name=BytesOutPerSec' + attribute: + Count: + metric_type: rate + alias: kafka.net.bytes_out.rate + - include: + domain: 'kafka.server' + bean: 'kafka.server:type=BrokerTopicMetrics,name=BytesInPerSec' + attribute: + Count: + metric_type: rate + alias: kafka.net.bytes_in.rate + - include: + domain: 'kafka.server' + bean: 'kafka.server:type=BrokerTopicMetrics,name=MessagesInPerSec' + attribute: + Count: + metric_type: rate + alias: kafka.messages_in.rate + - include: + domain: 'kafka.server' + bean: 'kafka.server:type=BrokerTopicMetrics,name=BytesRejectedPerSec' + attribute: + Count: + metric_type: rate + alias: kafka.net.bytes_rejected.rate + + # + # Per Topic Broker Stats + # + - include: + domain: '"kafka.server"' + bean_regex: '"kafka.server":type="BrokerTopicMetrics",name="(.*?)-MessagesInPerSec"' + attribute: + Count: + metric_type: rate + alias: kafka.topic.messages_in.rate + tags: + topic: $1 + + # + # Request timings + # + - include: + domain: 'kafka.server' + bean: 'kafka.server:type=BrokerTopicMetrics,name=FailedFetchRequestsPerSec' + attribute: + Count: + metric_type: rate + alias: kafka.request.fetch.failed.rate + - include: + domain: 'kafka.server' + bean: 'kafka.server:type=BrokerTopicMetrics,name=FailedProduceRequestsPerSec' + attribute: + Count: + metric_type: rate + alias: kafka.request.produce.failed.rate + - include: + domain: 'kafka.network' + bean: 'kafka.network:type=RequestMetrics,name=RequestsPerSec,request=Produce' + attribute: + Count: + metric_type: rate + alias: kafka.request.produce.rate + - include: + domain: 'kafka.network' + bean: 'kafka.network:type=RequestMetrics,name=TotalTimeMs,request=Produce' + attribute: + Mean: + metric_type: gauge + alias: kafka.request.produce.time.avg + 99thPercentile: + metric_type: gauge + alias: kafka.request.produce.time.99percentile + - include: + domain: 'kafka.network' + bean: 'kafka.network:type=RequestMetrics,name=RequestsPerSec,request=FetchConsumer' + attribute: + Count: + metric_type: rate + alias: kafka.request.fetch_consumer.rate + - include: + domain: 'kafka.network' + bean: 'kafka.network:type=RequestMetrics,name=RequestsPerSec,request=FetchFollower' + attribute: + Count: + metric_type: rate + alias: kafka.request.fetch_follower.rate + - include: + domain: 'kafka.network' + bean: 'kafka.network:type=RequestMetrics,name=TotalTimeMs,request=FetchConsumer' + attribute: + Mean: + metric_type: gauge + alias: kafka.request.fetch_consumer.time.avg + 99thPercentile: + metric_type: gauge + alias: kafka.request.fetch_consumer.time.99percentile + - include: + domain: 'kafka.network' + bean: 'kafka.network:type=RequestMetrics,name=TotalTimeMs,request=FetchFollower' + attribute: + Mean: + metric_type: gauge + alias: kafka.request.fetch_follower.time.avg + 99thPercentile: + metric_type: gauge + alias: kafka.request.fetch_follower.time.99percentile + - include: + domain: 'kafka.network' + bean: 'kafka.network:type=RequestMetrics,name=TotalTimeMs,request=UpdateMetadata' + attribute: + Mean: + metric_type: gauge + alias: kafka.request.update_metadata.time.avg + 99thPercentile: + metric_type: gauge + alias: kafka.request.update_metadata.time.99percentile + - include: + domain: 'kafka.network' + bean: 'kafka.network:type=RequestMetrics,name=TotalTimeMs,request=Metadata' + attribute: + Mean: + metric_type: gauge + alias: kafka.request.metadata.time.avg + 99thPercentile: + metric_type: gauge + alias: kafka.request.metadata.time.99percentile + - include: + domain: 'kafka.network' + bean: 'kafka.network:type=RequestMetrics,name=TotalTimeMs,request=Offsets' + attribute: + Mean: + metric_type: gauge + alias: kafka.request.offsets.time.avg + 99thPercentile: + metric_type: gauge + alias: kafka.request.offsets.time.99percentile + - include: + domain: 'kafka.server' + bean: 'kafka.server:type=KafkaRequestHandlerPool,name=RequestHandlerAvgIdlePercent' + attribute: + OneMinuteRate: + metric_type: gauge + alias: kafka.request.handler.avg.idle.pct.rate + - include: + domain: 'kafka.server' + bean: 'kafka.server:type=ProducerRequestPurgatory,name=PurgatorySize' + attribute: + Value: + metric_type: gauge + alias: kafka.request.producer_request_purgatory.size + - include: + domain: 'kafka.server' + bean: 'kafka.server:type=FetchRequestPurgatory,name=PurgatorySize' + attribute: + Value: + metric_type: gauge + alias: kafka.request.fetch_request_purgatory.size + + # + # Replication stats + # + - include: + domain: 'kafka.server' + bean: 'kafka.server:type=ReplicaManager,name=UnderReplicatedPartitions' + attribute: + Value: + metric_type: gauge + alias: kafka.replication.under_replicated_partitions + - include: + domain: 'kafka.server' + bean: 'kafka.server:type=ReplicaManager,name=IsrShrinksPerSec' + attribute: + Count: + metric_type: rate + alias: kafka.replication.isr_shrinks.rate + - include: + domain: 'kafka.server' + bean: 'kafka.server:type=ReplicaManager,name=IsrExpandsPerSec' + attribute: + Count: + metric_type: rate + alias: kafka.replication.isr_expands.rate + - include: + domain: 'kafka.controller' + bean: 'kafka.controller:type=ControllerStats,name=LeaderElectionRateAndTimeMs' + attribute: + Count: + metric_type: rate + alias: kafka.replication.leader_elections.rate + - include: + domain: 'kafka.controller' + bean: 'kafka.controller:type=ControllerStats,name=UncleanLeaderElectionsPerSec' + attribute: + Count: + metric_type: rate + alias: kafka.replication.unclean_leader_elections.rate + - include: + domain: 'kafka.controller' + bean: 'kafka.controller:type=KafkaController,name=OfflinePartitionsCount' + attribute: + Value: + metric_type: gauge + alias: kafka.replication.offline_partitions_count + - include: + domain: 'kafka.controller' + bean: 'kafka.controller:type=KafkaController,name=ActiveControllerCount' + attribute: + Value: + metric_type: gauge + alias: kafka.replication.active_controller_count + - include: + domain: 'kafka.server' + bean: 'kafka.server:type=ReplicaManager,name=PartitionCount' + attribute: + Value: + metric_type: gauge + alias: kafka.replication.partition_count + - include: + domain: 'kafka.server' + bean: 'kafka.server:type=ReplicaManager,name=LeaderCount' + attribute: + Value: + metric_type: gauge + alias: kafka.replication.leader_count + - include: + domain: 'kafka.server' + bean: 'kafka.server:type=ReplicaFetcherManager,name=MaxLag,clientId=Replica' + attribute: + Value: + metric_type: gauge + alias: kafka.replication.max_lag + + # + # Log flush stats + # + - include: + domain: 'kafka.log' + bean: 'kafka.log:type=LogFlushStats,name=LogFlushRateAndTimeMs' + attribute: + Count: + metric_type: rate + alias: kafka.log.flush_rate.rate diff --git a/dd-java-agent/agent-jmxfetch/src/main/resources/datadog/trace/agent/jmxfetch/metricconfigs/solr.yaml b/dd-java-agent/agent-jmxfetch/src/main/resources/datadog/trace/agent/jmxfetch/metricconfigs/solr.yaml new file mode 100644 index 0000000000..3fa966c5a0 --- /dev/null +++ b/dd-java-agent/agent-jmxfetch/src/main/resources/datadog/trace/agent/jmxfetch/metricconfigs/solr.yaml @@ -0,0 +1,200 @@ +# Default metrics collected by this check. You should not have to modify this. +jmx_metrics: + + # Solr versions >= 7.0 + - include: + category: SEARCHER + name: + - numDocs + scope: + - searcher + attribute: + Value: + alias: solr.searcher.numdocs + metric_type: gauge + + - include: + category: SEARCHER + name: maxDoc + scope: + - searcher + attribute: + Value: + alias: solr.searcher.maxdocs + metric_type: gauge + + - include: + category: SEARCHER + name: warmupTime + scope: + - searcher + attribute: + Value: + alias: solr.searcher.warmup + metric_type: gauge + + - include: + category: CACHE + name: documentCache + scope: + - searcher + attribute: + cumulative_lookups: + alias: solr.document_cache.lookups + metric_type: counter + cumulative_hits: + alias: solr.document_cache.hits + metric_type: counter + cumulative_inserts: + alias: solr.document_cache.inserts + metric_type: counter + cumulative_evictions: + alias: solr.document_cache.evictions + metric_type: counter + + - include: + category: CACHE + name: queryResultCache + scope: + - searcher + attribute: + cumulative_lookups: + alias: solr.query_result_cache.lookups + metric_type: counter + cumulative_hits: + alias: solr.query_result_cache.hits + metric_type: counter + cumulative_inserts: + alias: solr.query_result_cache.inserts + metric_type: counter + cumulative_evictions: + alias: solr.query_result_cache.evictions + metric_type: counter + + - include: + category: CACHE + name: filterCache + scope: + - searcher + attribute: + cumulative_lookups: + alias: solr.filter_cache.lookups + metric_type: counter + cumulative_hits: + alias: solr.filter_cache.hits + metric_type: counter + cumulative_inserts: + alias: solr.filter_cache.inserts + metric_type: counter + cumulative_evictions: + alias: solr.filter_cache.evictions + metric_type: counter + + - include: + category: QUERY + name: requests + attribute: + Count: + alias: solr.search_handler.requests + metric_type: counter + + - include: + category: QUERY + name: timeouts + attribute: + Count: + alias: solr.search_handler.timeouts + metric_type: counter + + - include: + category: QUERY + name: errors + attribute: + Count: + alias: solr.search_handler.errors + metric_type: counter + + - include: + category: QUERY + name: totalTime + attribute: + Count: + alias: solr.search_handler.time + metric_type: counter + + - include: + category: QUERY + name: requestTimes + attribute: + avgRequestsPerSecond: + alias: solr.search_handler.avg_requests_per_sec + metric_type: gauge + avgTimePerRequest: + alias: solr.search_handler.avg_time_per_req + metric_type: gauge + + + # Solr versions < 7.0 + - include: + type: searcher + attribute: + maxDoc: + alias: solr.searcher.maxdoc + metric_type: gauge + numDocs: + alias: solr.searcher.numdocs + metric_type: gauge + warmupTime: + alias: solr.searcher.warmup + metric_type: gauge + - include: + id: org.apache.solr.search.FastLRUCache + attribute: + cumulative_lookups: + alias: solr.cache.lookups + metric_type: counter + cumulative_hits: + alias: solr.cache.hits + metric_type: counter + cumulative_inserts: + alias: solr.cache.inserts + metric_type: counter + cumulative_evictions: + alias: solr.cache.evictions + metric_type: counter + - include: + id: org.apache.solr.search.LRUCache + attribute: + cumulative_lookups: + alias: solr.cache.lookups + metric_type: counter + cumulative_hits: + alias: solr.cache.hits + metric_type: counter + cumulative_inserts: + alias: solr.cache.inserts + metric_type: counter + cumulative_evictions: + alias: solr.cache.evictions + metric_type: counter + - include: + id: org.apache.solr.handler.component.SearchHandler + attribute: + errors: + alias: solr.search_handler.errors + metric_type: counter + requests: + alias: solr.search_handler.requests + metric_type: counter + timeouts: + alias: solr.search_handler.timeouts + metric_type: counter + totalTime: + alias: solr.search_handler.time + metric_type: counter + avgTimePerRequest: + alias: solr.search_handler.avg_time_per_req + metric_type: gauge + avgRequestsPerSecond: + alias: solr.search_handler.avg_requests_per_sec + metric_type: gauge diff --git a/dd-java-agent/agent-jmxfetch/src/main/resources/datadog/trace/agent/jmxfetch/metricconfigs/tomcat.yaml b/dd-java-agent/agent-jmxfetch/src/main/resources/datadog/trace/agent/jmxfetch/metricconfigs/tomcat.yaml new file mode 100644 index 0000000000..370cb90421 --- /dev/null +++ b/dd-java-agent/agent-jmxfetch/src/main/resources/datadog/trace/agent/jmxfetch/metricconfigs/tomcat.yaml @@ -0,0 +1,65 @@ +# Default metrics collected by this check. You should not have to modify this. +jmx_metrics: + - include: + type: ThreadPool + attribute: + maxThreads: + alias: tomcat.threads.max + metric_type: gauge + currentThreadCount: + alias: tomcat.threads.count + metric_type: gauge + currentThreadsBusy: + alias: tomcat.threads.busy + metric_type: gauge + - include: + type: GlobalRequestProcessor + attribute: + bytesSent: + alias: tomcat.bytes_sent + metric_type: counter + bytesReceived: + alias: tomcat.bytes_rcvd + metric_type: counter + errorCount: + alias: tomcat.error_count + metric_type: counter + requestCount: + alias: tomcat.request_count + metric_type: counter + maxTime: + alias: tomcat.max_time + metric_type: gauge + processingTime: + alias: tomcat.processing_time + metric_type: counter + - include: + j2eeType: Servlet + attribute: + processingTime: + alias: tomcat.servlet.processing_time + metric_type: counter + errorCount: + alias: tomcat.servlet.error_count + metric_type: counter + requestCount: + alias: tomcat.servlet.request_count + metric_type: counter + - include: + type: Cache + attribute: + accessCount: + alias: tomcat.cache.access_count + metric_type: counter + hitsCounts: + alias: tomcat.cache.hits_count + metric_type: counter + - include: + type: JspMonitor + attribute: + jspCount: + alias: tomcat.jsp.count + metric_type: counter + jspReloadCount: + alias: tomcat.jsp.reload_count + metric_type: counter