Adopt asynchronous helpers in Cassandra target script (#21)

This commit is contained in:
Ryan Fitzpatrick 2020-11-11 17:22:48 -05:00 committed by GitHub
parent 4449284563
commit 66f487eb1e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 123 additions and 249 deletions

View File

@ -28,7 +28,7 @@ otel.jmx.password = my-password
def storageLoadMBean = otel.mbean("org.apache.cassandra.metrics:type=Storage,name=Load")
otel.instrument(storageLoadMBean, "cassandra.storage.load",
"Size, in bytes, of the on disk data size this node manages",
"By", "Count", otel.&longValueRecorder
"By", "Count", otel.&longValueObserver
)
```

View File

@ -10,118 +10,118 @@ These metrics are sourced from Cassandra's exposed Dropwizard Metrics for each n
* Name: `cassandra.client.request.range_slice.latency.50p`
* Description: Token range read request latency - 50th percentile
* Unit: `µs`
* Instrument Type: DoubleUpDownCounter
* Instrument Type: DoubleValueObserver
* Name: `cassandra.client.request.range_slice.latency.99p`
* Description: Token range read request latency - 99th percentile
* Unit: `µs`
* Instrument Type: DoubleUpDownCounter
* Instrument Type: DoubleValueObserver
* Name: `cassandra.client.request.range_slice.latency.count`
* Description: Total token range read request latency
* Unit: `µs`
* Instrument Type: LongCounter
* Instrument Type: LongSumObserver
* Name: `cassandra.client.request.range_slice.latency.max`
* Description: Maximum token range read request latency
* Unit: `µs`
* Instrument Type: DoubleUpDownCounter
* Instrument Type: DoubleValueObserver
* Name: `cassandra.client.request.range_slice.timeout.count`
* Description: Number of token range read request timeouts encountered
* Unit: `1`
* Instrument Type: LongCounter
* Instrument Type: LongSumObserver
* Name: `cassandra.client.request.range_slice.unavailable.count`
* Description: Number of token range read request unavailable exceptions encountered
* Unit: `1`
* Instrument Type: LongCounter
* Instrument Type: LongSumObserver
* Name: `cassandra.client.request.read.latency.50p`
* Description: Standard read request latency - 50th percentile
* Unit: `µs`
* Instrument Type: DoubleUpDownCounter
* Instrument Type: DoubleValueObserver
* Name: `cassandra.client.request.read.latency.99p`
* Description: Standard read request latency - 99th percentile
* Unit: `µs`
* Instrument Type: DoubleUpDownCounter
* Instrument Type: DoubleValueObserver
* Name: `cassandra.client.request.read.latency.count`
* Description: Total standard read request latency
* Unit: `µs`
* Instrument Type: LongCounter
* Instrument Type: LongSumObserver
* Name: `cassandra.client.request.read.latency.max`
* Description: Maximum standard read request latency
* Unit: `µs`
* Instrument Type: DoubleUpDownCounter
* Instrument Type: DoubleValueObserver
* Name: `cassandra.client.request.read.timeout.count`
* Description: Number of standard read request timeouts encountered
* Unit: `1`
* Instrument Type: LongCounter
* Instrument Type: LongSumObserver
* Name: `cassandra.client.request.read.unavailable.count`
* Description: Number of standard read request unavailable exceptions encountered
* Unit: `1`
* Instrument Type: LongCounter
* Instrument Type: LongSumObserver
* Name: `cassandra.client.request.write.latency.50p`
* Description: Regular write request latency - 50th percentile
* Unit: `µs`
* Instrument Type: DoubleUpDownCounter
* Instrument Type: DoubleValueObserver
* Name: `cassandra.client.request.write.latency.99p`
* Description: Regular write request latency - 99th percentile
* Unit: `µs`
* Instrument Type: DoubleUpDownCounter
* Instrument Type: DoubleValueObserver
* Name: `cassandra.client.request.write.latency.count`
* Description: Total regular write request latency
* Unit: `µs`
* Instrument Type: LongCounter
* Instrument Type: LongSumObserver
* Name: `cassandra.client.request.write.latency.max`
* Description: Maximum regular write request latency
* Unit: `µs`
* Instrument Type: DoubleUpDownCounter
* Instrument Type: DoubleValueObserver
* Name: `cassandra.client.request.write.timeout.count`
* Description: Number of regular write request timeouts encountered
* Unit: `1`
* Instrument Type: LongCounter
* Instrument Type: LongSumObserver
* Name: `cassandra.client.request.write.unavailable.count`
* Description: Number of regular write request unavailable exceptions encountered
* Unit: `1`
* Instrument Type: LongCounter
* Instrument Type: LongSumObserver
### Compaction Metrics
* Name: `cassandra.compaction.tasks.completed`
* Description: Number of completed compactions since server [re]start
* Unit: `1`
* Instrument Type: LongCounter
* Instrument Type: LongSumObserver
* Name: `cassandra.compaction.tasks.pending`
* Description: Estimated number of compactions remaining to perform
* Unit: `1`
* Instrument Type: LongUpDownCounter
* Instrument Type: LongValueObserver
### Storage Metrics
* Name: `cassandra.storage.load.count`
* Description: Size of the on disk data size this node manages
* Unit: `by`
* Instrument Type: LongUpDownCounter
* Instrument Type: LongValueObserver
* Name: `cassandra.storage.total_hints.count`
* Description: Number of hint messages written to this node since [re]start
* Unit: `1`
* Instrument Type: LongCounter
* Instrument Type: LongSumObserver
* Name: `cassandra.storage.total_hints.in_progress.count`
* Description: Number of hints attempting to be sent currently
* Unit: `1`
* Instrument Type: LongUpDownCounter
* Instrument Type: LongValueObserver

View File

@ -14,271 +14,138 @@
* limitations under the License.
*/
class CassandraMBean {
// Necessary to have script-bound `otel` and `log` in class scope
protected Binding sb
protected GroovyMBean mbean
private String objectName
CassandraMBean(Binding scriptBinding, objectName) {
sb = scriptBinding
this.objectName = objectName
}
def fetch() {
def mbeans = sb.otel.queryJmx(objectName)
if (mbeans.size() == 0) {
sb.log.warning("CassandraMBean.fetch(): Failed to fetch MBean ${objectName}.")
} else {
sb.log.fine("CassandraMBean.fetch(): Fetched ${mbeans.size()} MBeans - ${mbeans}")
mbean = mbeans.first()
}
}
def getAttribute(String attribute) {
if (mbean == null) {
return null
}
return mbean.getProperty(attribute)
}
}
class CassandraMetric {
private CassandraMBean cassandraMBean
private Binding sb
private String instrumentName
private String description
private String unit
private String attribute
private Closure instrument
CassandraMetric(CassandraMBean cassandraMBean, String instrumentName, String description, String unit, String attribute, Closure instrument) {
this.cassandraMBean = cassandraMBean
this.sb = cassandraMBean.sb
this.instrumentName = instrumentName
this.description = description
this.unit = unit
this.attribute = attribute
this.instrument = instrument
}
def update() {
def value = cassandraMBean.getAttribute(attribute)
if (value == null) {
sb.log.warning("No valid value for ${instrumentName} - ${cassandraMBean}.${attribute}" as String)
return
}
def inst = instrument(instrumentName, description, unit)
sb.log.fine("Recording ${instrumentName} - ${inst} w/ ${value}")
inst.add(value)
}
}
def cassandraMetrics = "org.apache.cassandra.metrics"
def clientRequest = "${cassandraMetrics}:type=ClientRequest"
def clientRequestRangeSlice = "${clientRequest},scope=RangeSlice"
def clientRequestRead = "${clientRequest},scope=Read"
def clientRequestWrite = "${clientRequest},scope=Write"
def compaction = "${cassandraMetrics}:type=Compaction"
def storage = "${cassandraMetrics}:type=Storage"
def clientRequestRangeSliceLatency = new CassandraMBean(binding, "${clientRequestRangeSlice},name=Latency")
def clientRequestRangeSliceTimeouts = new CassandraMBean(binding, "${clientRequestRangeSlice},name=Timeouts")
def clientRequestRangeSliceUnavailables = new CassandraMBean(binding, "${clientRequestRangeSlice},name=Unavailables")
def clientRequestReadLatency = new CassandraMBean(binding, "${clientRequestRead},name=Latency")
def clientRequestReadTimeouts = new CassandraMBean(binding, "${clientRequestRead},name=Timeouts")
def clientRequestReadUnavailables = new CassandraMBean(binding, "${clientRequestRead},name=Unavailables")
def clientRequestWriteLatency = new CassandraMBean(binding, "${clientRequestWrite},name=Latency")
def clientRequestWriteTimeouts = new CassandraMBean(binding, "${clientRequestWrite},name=Timeouts")
def clientRequestWriteUnavailables = new CassandraMBean(binding, "${clientRequestWrite},name=Unavailables")
def compactionCompletedTasks = new CassandraMBean(binding, "${compaction},name=CompletedTasks")
def compactionPendingTasks = new CassandraMBean(binding, "${compaction},name=PendingTasks")
def storageLoad = new CassandraMBean(binding, "${storage},name=Load")
def storageTotalHints = new CassandraMBean(binding, "${storage},name=TotalHints")
def storageTotalHintsInProgress = new CassandraMBean(binding, "${storage},name=TotalHintsInProgress")
def clientRequestRangeSliceLatency50p = new CassandraMetric(
clientRequestRangeSliceLatency,
def clientRequestRangeSliceLatency = otel.mbean("${clientRequestRangeSlice},name=Latency")
otel.instrument(clientRequestRangeSliceLatency,
"cassandra.client.request.range_slice.latency.50p",
"Token range read request latency - 50th percentile", "µs", "50thPercentile",
otel.&doubleUpDownCounter)
otel.&doubleValueObserver)
def clientRequestRangeSliceLatency99p = new CassandraMetric(
clientRequestRangeSliceLatency,
otel.instrument(clientRequestRangeSliceLatency,
"cassandra.client.request.range_slice.latency.99p",
"Token range read request latency - 99th percentile", "µs", "99thPercentile",
otel.&doubleUpDownCounter)
otel.&doubleValueObserver)
def clientRequestRangeSliceLatencyCount = new CassandraMetric(
clientRequestRangeSliceLatency,
otel.instrument(clientRequestRangeSliceLatency,
"cassandra.client.request.range_slice.latency.count",
"Total token range read request latency", "µs", "Count",
otel.&longCounter)
otel.&longSumObserver)
def clientRequestRangeSliceLatencyMax = new CassandraMetric(
clientRequestRangeSliceLatency,
otel.instrument(clientRequestRangeSliceLatency,
"cassandra.client.request.range_slice.latency.max",
"Maximum token range read request latency", "µs", "Max",
otel.&doubleUpDownCounter)
otel.&doubleValueObserver)
def clientRequestRangeSliceTimeoutCount = new CassandraMetric(
clientRequestRangeSliceTimeouts,
def clientRequestRangeSliceTimeouts = otel.mbean("${clientRequestRangeSlice},name=Timeouts")
otel.instrument(clientRequestRangeSliceTimeouts,
"cassandra.client.request.range_slice.timeout.count",
"Number of token range read request timeouts encountered", "1", "Count",
otel.&longCounter)
otel.&longSumObserver)
def clientRequestRangeSliceUnavailableCount = new CassandraMetric(
clientRequestRangeSliceUnavailables,
def clientRequestRangeSliceUnavailables = otel.mbean("${clientRequestRangeSlice},name=Unavailables")
otel.instrument(clientRequestRangeSliceUnavailables,
"cassandra.client.request.range_slice.unavailable.count",
"Number of token range read request unavailable exceptions encountered", "1", "Count",
otel.&longCounter)
otel.&longSumObserver)
def clientRequestReadLatency50p = new CassandraMetric(clientRequestReadLatency,
def clientRequestRead = "${clientRequest},scope=Read"
def clientRequestReadLatency = otel.mbean("${clientRequestRead},name=Latency")
otel.instrument(clientRequestReadLatency,
"cassandra.client.request.read.latency.50p",
"Standard read request latency - 50th percentile", "µs", "50thPercentile",
otel.&doubleUpDownCounter)
otel.&doubleValueObserver)
def clientRequestReadLatency99p = new CassandraMetric(
clientRequestReadLatency,
otel.instrument(clientRequestReadLatency,
"cassandra.client.request.read.latency.99p",
"Standard read request latency - 99th percentile", "µs", "99thPercentile",
otel.&doubleUpDownCounter)
otel.&doubleValueObserver)
def clientRequestReadLatencyCount = new CassandraMetric(
clientRequestReadLatency,
otel.instrument(clientRequestReadLatency,
"cassandra.client.request.read.latency.count",
"Total standard read request latency", "µs", "Count",
otel.&longCounter)
otel.&longSumObserver)
def clientRequestReadLatencyMax = new CassandraMetric(
clientRequestReadLatency,
otel.instrument(clientRequestReadLatency,
"cassandra.client.request.read.latency.max",
"Maximum standard read request latency", "µs", "Max",
otel.&doubleUpDownCounter)
otel.&doubleValueObserver)
def clientRequestReadTimeoutCount = new CassandraMetric(
clientRequestReadTimeouts,
def clientRequestReadTimeouts = otel.mbean("${clientRequestRead},name=Timeouts")
otel.instrument(clientRequestReadTimeouts,
"cassandra.client.request.read.timeout.count",
"Number of standard read request timeouts encountered", "1", "Count",
otel.&longCounter)
otel.&longSumObserver)
def clientRequestReadUnavailableCount = new CassandraMetric(
clientRequestReadUnavailables,
def clientRequestReadUnavailables = otel.mbean("${clientRequestRead},name=Unavailables")
otel.instrument(clientRequestReadUnavailables,
"cassandra.client.request.read.unavailable.count",
"Number of standard read request unavailable exceptions encountered", "1", "Count",
otel.&longCounter)
otel.&longSumObserver)
def clientRequestWriteLatency50p = new CassandraMetric(
clientRequestWriteLatency,
def clientRequestWrite = "${clientRequest},scope=Write"
def clientRequestWriteLatency = otel.mbean("${clientRequestWrite},name=Latency")
otel.instrument(clientRequestWriteLatency,
"cassandra.client.request.write.latency.50p",
"Regular write request latency - 50th percentile", "µs", "50thPercentile",
otel.&doubleUpDownCounter)
otel.&doubleValueObserver)
def clientRequestWriteLatency99p = new CassandraMetric(
clientRequestWriteLatency,
otel.instrument(clientRequestWriteLatency,
"cassandra.client.request.write.latency.99p",
"Regular write request latency - 99th percentile", "µs", "99thPercentile",
otel.&doubleUpDownCounter)
otel.&doubleValueObserver)
def clientRequestWriteLatencyCount = new CassandraMetric(
clientRequestWriteLatency,
otel.instrument(clientRequestWriteLatency,
"cassandra.client.request.write.latency.count",
"Total regular write request latency", "µs", "Count",
otel.&longCounter)
otel.&longSumObserver)
def clientRequestWriteLatencyMax = new CassandraMetric(
clientRequestWriteLatency,
otel.instrument(clientRequestWriteLatency,
"cassandra.client.request.write.latency.max",
"Maximum regular write request latency", "µs", "Max",
otel.&doubleUpDownCounter)
otel.&doubleValueObserver)
def clientRequestWriteTimeoutCount = new CassandraMetric(
clientRequestWriteTimeouts,
def clientRequestWriteTimeouts = otel.mbean("${clientRequestWrite},name=Timeouts")
otel.instrument(clientRequestWriteTimeouts,
"cassandra.client.request.write.timeout.count",
"Number of regular write request timeouts encountered", "1", "Count",
otel.&longCounter)
otel.&longSumObserver)
def clientRequestWriteUnavailableCount = new CassandraMetric(
clientRequestWriteUnavailables,
def clientRequestWriteUnavailables = otel.mbean("${clientRequestWrite},name=Unavailables")
otel.instrument(clientRequestWriteUnavailables,
"cassandra.client.request.write.unavailable.count",
"Number of regular write request unavailable exceptions encountered", "1", "Count",
otel.&longCounter)
otel.&longSumObserver)
def storageLoadCount = new CassandraMetric(
storageLoad,
def storage = "${cassandraMetrics}:type=Storage"
def storageLoad = otel.mbean("${storage},name=Load")
otel.instrument(storageLoad,
"cassandra.storage.load.count",
"Size of the on disk data size this node manages", "by", "Count",
otel.&longUpDownCounter)
otel.&longSumObserver)
def storageTotalHintsCount = new CassandraMetric(
storageTotalHints,
def storageTotalHints = otel.mbean("${storage},name=TotalHints")
otel.instrument(storageTotalHints,
"cassandra.storage.total_hints.count",
"Number of hint messages written to this node since [re]start", "1", "Count",
otel.&longCounter)
otel.&longSumObserver)
def storageTotalHintsInProgressCount = new CassandraMetric(
storageTotalHintsInProgress,
def storageTotalHintsInProgress = otel.mbean("${storage},name=TotalHintsInProgress")
otel.instrument(storageTotalHintsInProgress,
"cassandra.storage.total_hints.in_progress.count",
"Number of hints attempting to be sent currently", "1", "Count",
otel.&longUpDownCounter)
otel.&longSumObserver)
def compactionTasksPending = new CassandraMetric(
compactionPendingTasks,
def compaction = "${cassandraMetrics}:type=Compaction"
def compactionPendingTasks = otel.mbean("${compaction},name=PendingTasks")
otel.instrument(compactionPendingTasks,
"cassandra.compaction.tasks.pending",
"Estimated number of compactions remaining to perform", "1", "Value",
otel.&longUpDownCounter)
otel.&longValueObserver)
def compactionTasksCompleted = new CassandraMetric(
compactionCompletedTasks,
def compactionCompletedTasks = otel.mbean("${compaction},name=CompletedTasks")
otel.instrument(compactionCompletedTasks,
"cassandra.compaction.tasks.completed",
"Number of completed compactions since server [re]start", "1", "Value",
otel.&longCounter)
[
clientRequestRangeSliceLatency,
clientRequestRangeSliceTimeouts,
clientRequestRangeSliceUnavailables,
clientRequestReadLatency,
clientRequestReadTimeouts,
clientRequestReadUnavailables,
clientRequestWriteLatency,
clientRequestWriteTimeouts,
clientRequestWriteUnavailables,
compactionCompletedTasks,
compactionPendingTasks,
storageLoad,
storageTotalHints,
storageTotalHintsInProgress,
].each {
it.fetch()
}
[
clientRequestRangeSliceLatency50p,
clientRequestRangeSliceLatency99p,
clientRequestRangeSliceLatencyCount,
clientRequestRangeSliceLatencyMax,
clientRequestRangeSliceTimeoutCount,
clientRequestRangeSliceUnavailableCount,
clientRequestReadLatency50p,
clientRequestReadLatency99p,
clientRequestReadLatencyCount,
clientRequestReadLatencyMax,
clientRequestReadTimeoutCount,
clientRequestReadUnavailableCount,
clientRequestWriteLatency50p,
clientRequestWriteLatency99p,
clientRequestWriteLatencyCount,
clientRequestWriteLatencyMax,
clientRequestWriteTimeoutCount,
clientRequestWriteUnavailableCount,
compactionTasksCompleted,
compactionTasksPending,
storageLoadCount,
storageTotalHintsCount,
storageTotalHintsInProgressCount,
].each {
it.update()
}
otel.&longSumObserver)

View File

@ -16,7 +16,8 @@
package io.opentelemetry.contrib.jmxmetrics
import io.opentelemetry.proto.metrics.v1.DoubleSum
import io.opentelemetry.proto.metrics.v1.DoubleGauge
import io.opentelemetry.proto.metrics.v1.IntGauge
import io.opentelemetry.proto.metrics.v1.IntSum
import io.opentelemetry.proto.common.v1.InstrumentationLibrary
@ -71,139 +72,139 @@ class CassandraIntegrationTests extends OtlpIntegrationTest {
'cassandra.client.request.range_slice.latency.50p',
'Token range read request latency - 50th percentile',
'µs',
'double',
DoubleGauge
],
[
'cassandra.client.request.range_slice.latency.99p',
'Token range read request latency - 99th percentile',
'µs',
'double',
DoubleGauge
],
[
'cassandra.client.request.range_slice.latency.count',
'Total token range read request latency',
'µs',
'int',
IntSum
],
[
'cassandra.client.request.range_slice.latency.max',
'Maximum token range read request latency',
'µs',
'double',
DoubleGauge,
],
[
'cassandra.client.request.range_slice.timeout.count',
'Number of token range read request timeouts encountered',
'1',
'int',
IntSum,
],
[
'cassandra.client.request.range_slice.unavailable.count',
'Number of token range read request unavailable exceptions encountered',
'1',
'int',
IntSum,
],
[
'cassandra.client.request.read.latency.50p',
'Standard read request latency - 50th percentile',
'µs',
'double',
DoubleGauge,
],
[
'cassandra.client.request.read.latency.99p',
'Standard read request latency - 99th percentile',
'µs',
'double',
DoubleGauge,
],
[
'cassandra.client.request.read.latency.count',
'Total standard read request latency',
'µs',
'int',
IntSum,
],
[
'cassandra.client.request.read.latency.max',
'Maximum standard read request latency',
'µs',
'double',
DoubleGauge,
],
[
'cassandra.client.request.read.timeout.count',
'Number of standard read request timeouts encountered',
'1',
'int',
IntSum,
],
[
'cassandra.client.request.read.unavailable.count',
'Number of standard read request unavailable exceptions encountered',
'1',
'int',
IntSum,
],
[
'cassandra.client.request.write.latency.50p',
'Regular write request latency - 50th percentile',
'µs',
'double',
DoubleGauge,
],
[
'cassandra.client.request.write.latency.99p',
'Regular write request latency - 99th percentile',
'µs',
'double',
DoubleGauge,
],
[
'cassandra.client.request.write.latency.count',
'Total regular write request latency',
'µs',
'int',
IntSum,
],
[
'cassandra.client.request.write.latency.max',
'Maximum regular write request latency',
'µs',
'double',
DoubleGauge,
],
[
'cassandra.client.request.write.timeout.count',
'Number of regular write request timeouts encountered',
'1',
'int',
IntSum,
],
[
'cassandra.client.request.write.unavailable.count',
'Number of regular write request unavailable exceptions encountered',
'1',
'int',
IntSum,
],
[
'cassandra.compaction.tasks.completed',
'Number of completed compactions since server [re]start',
'1',
'int',
IntSum,
],
[
'cassandra.compaction.tasks.pending',
'Estimated number of compactions remaining to perform',
'1',
'int',
IntGauge,
],
[
'cassandra.storage.load.count',
'Size of the on disk data size this node manages',
'by',
'int',
IntSum,
],
[
'cassandra.storage.total_hints.count',
'Number of hint messages written to this node since [re]start',
'1',
'int',
IntSum,
],
[
'cassandra.storage.total_hints.in_progress.count',
'Number of hints attempting to be sent currently',
'1',
'int',
IntSum,
],
].eachWithIndex{ item, index ->
Metric metric = metrics.get(index)
@ -212,13 +213,19 @@ class CassandraIntegrationTests extends OtlpIntegrationTest {
assert metric.unit == item[2]
def datapoint
switch(item[3]) {
case 'double':
assert metric.hasDoubleSum()
DoubleSum datapoints = metric.doubleSum
case DoubleGauge:
assert metric.hasDoubleGauge()
DoubleGauge datapoints = metric.doubleGauge
assert datapoints.dataPointsCount == 1
datapoint = datapoints.getDataPoints(0)
break
case 'int':
case IntGauge:
assert metric.hasIntGauge()
IntGauge datapoints = metric.intGauge
assert datapoints.dataPointsCount == 1
datapoint = datapoints.getDataPoints(0)
break
case IntSum:
assert metric.hasIntSum()
IntSum datapoints = metric.intSum
assert datapoints.dataPointsCount == 1