tikv
/
client-go
mirror of https://github.com/tikv/client-go.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

add const label for metrics (#802) 

Signed-off-by: zeminzhou <zhouzemin@pingcap.com>
Co-authored-by: disksing <i@disksing.com>
This commit is contained in:
zzm 2023-06-07 13:48:17 +08:00 committed by GitHub
parent 28247160f4
commit cf07be2ba0
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 345 additions and 279 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

624
metrics/metrics.go
View File

@ -128,536 +128,602 @@ const (
LblDirection = "direction" LblDirection = "direction"
) )
func initMetrics(namespace, subsystem string) { func initMetrics(namespace, subsystem string, constLabels prometheus.Labels) {
TiKVTxnCmdHistogram = prometheus.NewHistogramVec( TiKVTxnCmdHistogram = prometheus.NewHistogramVec(
prometheus.HistogramOpts{ prometheus.HistogramOpts{
Namespace: namespace, Namespace: namespace,
Subsystem: subsystem, Subsystem: subsystem,
Name: "txn_cmd_duration_seconds", Name: "txn_cmd_duration_seconds",
Help: "Bucketed histogram of processing time of txn cmds.", Help: "Bucketed histogram of processing time of txn cmds.",
Buckets: prometheus.ExponentialBuckets(0.0005, 2, 29), // 0.5ms ~ 1.5days Buckets: prometheus.ExponentialBuckets(0.0005, 2, 29), // 0.5ms ~ 1.5days
ConstLabels: constLabels,
}, []string{LblType, LblScope}) }, []string{LblType, LblScope})
TiKVBackoffHistogram = prometheus.NewHistogramVec( TiKVBackoffHistogram = prometheus.NewHistogramVec(
prometheus.HistogramOpts{ prometheus.HistogramOpts{
Namespace: namespace, Namespace: namespace,
Subsystem: subsystem, Subsystem: subsystem,
Name: "backoff_seconds", Name: "backoff_seconds",
Help: "total backoff seconds of a single backoffer.", Help: "total backoff seconds of a single backoffer.",
Buckets: prometheus.ExponentialBuckets(0.0005, 2, 29), // 0.5ms ~ 1.5days Buckets: prometheus.ExponentialBuckets(0.0005, 2, 29), // 0.5ms ~ 1.5days
ConstLabels: constLabels,
}, []string{LblType}) }, []string{LblType})
TiKVSendReqHistogram = prometheus.NewHistogramVec( TiKVSendReqHistogram = prometheus.NewHistogramVec(
prometheus.HistogramOpts{ prometheus.HistogramOpts{
Namespace: namespace, Namespace: namespace,
Subsystem: subsystem, Subsystem: subsystem,
Name: "request_seconds", Name: "request_seconds",
Help: "Bucketed histogram of sending request duration.", Help: "Bucketed histogram of sending request duration.",
Buckets: prometheus.ExponentialBuckets(0.0005, 2, 29), // 0.5ms ~ 1.5days Buckets: prometheus.ExponentialBuckets(0.0005, 2, 29), // 0.5ms ~ 1.5days
ConstLabels: constLabels,
}, []string{LblType, LblStore, LblStaleRead, LblScope}) }, []string{LblType, LblStore, LblStaleRead, LblScope})
TiKVSendReqCounter = prometheus.NewCounterVec( TiKVSendReqCounter = prometheus.NewCounterVec(
prometheus.CounterOpts{ prometheus.CounterOpts{
Namespace: namespace, Namespace: namespace,
Subsystem: subsystem, Subsystem: subsystem,
Name: "request_counter", Name: "request_counter",
Help: "Counter of sending request with multi dimensions.", Help: "Counter of sending request with multi dimensions.",
ConstLabels: constLabels,
}, []string{LblType, LblStore, LblStaleRead, LblSource, LblScope}) }, []string{LblType, LblStore, LblStaleRead, LblSource, LblScope})
TiKVSendReqTimeCounter = prometheus.NewCounterVec( TiKVSendReqTimeCounter = prometheus.NewCounterVec(
prometheus.CounterOpts{ prometheus.CounterOpts{
Namespace: namespace, Namespace: namespace,
Subsystem: subsystem, Subsystem: subsystem,
Name: "request_time_counter", Name: "request_time_counter",
Help: "Counter of request time with multi dimensions.", Help: "Counter of request time with multi dimensions.",
ConstLabels: constLabels,
}, []string{LblType, LblStore, LblStaleRead, LblSource, LblScope}) }, []string{LblType, LblStore, LblStaleRead, LblSource, LblScope})
TiKVRPCNetLatencyHistogram = prometheus.NewHistogramVec( TiKVRPCNetLatencyHistogram = prometheus.NewHistogramVec(
prometheus.HistogramOpts{ prometheus.HistogramOpts{
Namespace: namespace, Namespace: namespace,
Subsystem: subsystem, Subsystem: subsystem,
Name: "rpc_net_latency_seconds", Name: "rpc_net_latency_seconds",
Help: "Bucketed histogram of time difference between TiDB and TiKV.", Help: "Bucketed histogram of time difference between TiDB and TiKV.",
Buckets: prometheus.ExponentialBuckets(5e-5, 2, 18), // 50us ~ 6.5s Buckets: prometheus.ExponentialBuckets(5e-5, 2, 18), // 50us ~ 6.5s
ConstLabels: constLabels,
}, []string{LblStore, LblScope}) }, []string{LblStore, LblScope})
TiKVCoprocessorHistogram = prometheus.NewHistogramVec( TiKVCoprocessorHistogram = prometheus.NewHistogramVec(
prometheus.HistogramOpts{ prometheus.HistogramOpts{
Namespace: namespace, Namespace: namespace,
Subsystem: subsystem, Subsystem: subsystem,
Name: "cop_duration_seconds", Name: "cop_duration_seconds",
Help: "Run duration of a single coprocessor task, includes backoff time.", Help: "Run duration of a single coprocessor task, includes backoff time.",
Buckets: prometheus.ExponentialBuckets(0.0005, 2, 29), // 0.5ms ~ 1.5days Buckets: prometheus.ExponentialBuckets(0.0005, 2, 29), // 0.5ms ~ 1.5days
ConstLabels: constLabels,
}, []string{LblStore, LblStaleRead, LblScope}) }, []string{LblStore, LblStaleRead, LblScope})
TiKVLockResolverCounter = prometheus.NewCounterVec( TiKVLockResolverCounter = prometheus.NewCounterVec(
prometheus.CounterOpts{ prometheus.CounterOpts{
Namespace: namespace, Namespace: namespace,
Subsystem: subsystem, Subsystem: subsystem,
Name: "lock_resolver_actions_total", Name: "lock_resolver_actions_total",
Help: "Counter of lock resolver actions.", Help: "Counter of lock resolver actions.",
ConstLabels: constLabels,
}, []string{LblType}) }, []string{LblType})
TiKVRegionErrorCounter = prometheus.NewCounterVec( TiKVRegionErrorCounter = prometheus.NewCounterVec(
prometheus.CounterOpts{ prometheus.CounterOpts{
Namespace: namespace, Namespace: namespace,
Subsystem: subsystem, Subsystem: subsystem,
Name: "region_err_total", Name: "region_err_total",
Help: "Counter of region errors.", Help: "Counter of region errors.",
ConstLabels: constLabels,
}, []string{LblType, LblScope}) }, []string{LblType, LblScope})
TiKVTxnWriteKVCountHistogram = prometheus.NewHistogramVec( TiKVTxnWriteKVCountHistogram = prometheus.NewHistogramVec(
prometheus.HistogramOpts{ prometheus.HistogramOpts{
Namespace: namespace, Namespace: namespace,
Subsystem: subsystem, Subsystem: subsystem,
Name: "txn_write_kv_num", Name: "txn_write_kv_num",
Help: "Count of kv pairs to write in a transaction.", Help: "Count of kv pairs to write in a transaction.",
Buckets: prometheus.ExponentialBuckets(1, 4, 17), // 1 ~ 4G Buckets: prometheus.ExponentialBuckets(1, 4, 17), // 1 ~ 4G
ConstLabels: constLabels,
}, []string{LblScope}) }, []string{LblScope})
TiKVTxnWriteSizeHistogram = prometheus.NewHistogramVec( TiKVTxnWriteSizeHistogram = prometheus.NewHistogramVec(
prometheus.HistogramOpts{ prometheus.HistogramOpts{
Namespace: namespace, Namespace: namespace,
Subsystem: subsystem, Subsystem: subsystem,
Name: "txn_write_size_bytes", Name: "txn_write_size_bytes",
Help: "Size of kv pairs to write in a transaction.", Help: "Size of kv pairs to write in a transaction.",
Buckets: prometheus.ExponentialBuckets(16, 4, 17), // 16Bytes ~ 64GB Buckets: prometheus.ExponentialBuckets(16, 4, 17), // 16Bytes ~ 64GB
ConstLabels: constLabels,
}, []string{LblScope}) }, []string{LblScope})
TiKVRawkvCmdHistogram = prometheus.NewHistogramVec( TiKVRawkvCmdHistogram = prometheus.NewHistogramVec(
prometheus.HistogramOpts{ prometheus.HistogramOpts{
Namespace: namespace, Namespace: namespace,
Subsystem: subsystem, Subsystem: subsystem,
Name: "rawkv_cmd_seconds", Name: "rawkv_cmd_seconds",
Help: "Bucketed histogram of processing time of rawkv cmds.", Help: "Bucketed histogram of processing time of rawkv cmds.",
Buckets: prometheus.ExponentialBuckets(0.0005, 2, 29), // 0.5ms ~ 1.5days Buckets: prometheus.ExponentialBuckets(0.0005, 2, 29), // 0.5ms ~ 1.5days
ConstLabels: constLabels,
}, []string{LblType}) }, []string{LblType})
TiKVRawkvSizeHistogram = prometheus.NewHistogramVec( TiKVRawkvSizeHistogram = prometheus.NewHistogramVec(
prometheus.HistogramOpts{ prometheus.HistogramOpts{
Namespace: namespace, Namespace: namespace,
Subsystem: subsystem, Subsystem: subsystem,
Name: "rawkv_kv_size_bytes", Name: "rawkv_kv_size_bytes",
Help: "Size of key/value to put, in bytes.", Help: "Size of key/value to put, in bytes.",
Buckets: prometheus.ExponentialBuckets(1, 2, 30), // 1Byte ~ 512MB Buckets: prometheus.ExponentialBuckets(1, 2, 30), // 1Byte ~ 512MB
ConstLabels: constLabels,
}, []string{LblType}) }, []string{LblType})
TiKVTxnRegionsNumHistogram = prometheus.NewHistogramVec( TiKVTxnRegionsNumHistogram = prometheus.NewHistogramVec(
prometheus.HistogramOpts{ prometheus.HistogramOpts{
Namespace: namespace, Namespace: namespace,
Subsystem: subsystem, Subsystem: subsystem,
Name: "txn_regions_num", Name: "txn_regions_num",
Help: "Number of regions in a transaction.", Help: "Number of regions in a transaction.",
Buckets: prometheus.ExponentialBuckets(1, 2, 25), // 1 ~ 16M Buckets: prometheus.ExponentialBuckets(1, 2, 25), // 1 ~ 16M
ConstLabels: constLabels,
}, []string{LblType, LblScope}) }, []string{LblType, LblScope})
TiKVLoadSafepointCounter = prometheus.NewCounterVec( TiKVLoadSafepointCounter = prometheus.NewCounterVec(
prometheus.CounterOpts{ prometheus.CounterOpts{
Namespace: namespace, Namespace: namespace,
Subsystem: subsystem, Subsystem: subsystem,
Name: "load_safepoint_total", Name: "load_safepoint_total",
Help: "Counter of load safepoint.", Help: "Counter of load safepoint.",
ConstLabels: constLabels,
}, []string{LblType}) }, []string{LblType})
TiKVSecondaryLockCleanupFailureCounter = prometheus.NewCounterVec( TiKVSecondaryLockCleanupFailureCounter = prometheus.NewCounterVec(
prometheus.CounterOpts{ prometheus.CounterOpts{
Namespace: namespace, Namespace: namespace,
Subsystem: subsystem, Subsystem: subsystem,
Name: "lock_cleanup_task_total", Name: "lock_cleanup_task_total",
Help: "failure statistic of secondary lock cleanup task.", Help: "failure statistic of secondary lock cleanup task.",
ConstLabels: constLabels,
}, []string{LblType}) }, []string{LblType})
TiKVRegionCacheCounter = prometheus.NewCounterVec( TiKVRegionCacheCounter = prometheus.NewCounterVec(
prometheus.CounterOpts{ prometheus.CounterOpts{
Namespace: namespace, Namespace: namespace,
Subsystem: subsystem, Subsystem: subsystem,
Name: "region_cache_operations_total", Name: "region_cache_operations_total",
Help: "Counter of region cache.", Help: "Counter of region cache.",
ConstLabels: constLabels,
}, []string{LblType, LblResult}) }, []string{LblType, LblResult})
TiKVLoadRegionCacheHistogram = prometheus.NewHistogramVec( TiKVLoadRegionCacheHistogram = prometheus.NewHistogramVec(
prometheus.HistogramOpts{ prometheus.HistogramOpts{
Namespace: namespace, Namespace: namespace,
Subsystem: subsystem, Subsystem: subsystem,
Name: "load_region_cache_seconds", Name: "load_region_cache_seconds",
Help: "Load region information duration", Help: "Load region information duration",
Buckets: prometheus.ExponentialBuckets(0.0001, 2, 20), // 0.1ms ~ 52s Buckets: prometheus.ExponentialBuckets(0.0001, 2, 20), // 0.1ms ~ 52s
ConstLabels: constLabels,
}, []string{LblType}) }, []string{LblType})
TiKVLocalLatchWaitTimeHistogram = prometheus.NewHistogram( TiKVLocalLatchWaitTimeHistogram = prometheus.NewHistogram(
prometheus.HistogramOpts{ prometheus.HistogramOpts{
Namespace: namespace, Namespace: namespace,
Subsystem: subsystem, Subsystem: subsystem,
Name: "local_latch_wait_seconds", Name: "local_latch_wait_seconds",
Help: "Wait time of a get local latch.", Help: "Wait time of a get local latch.",
Buckets: prometheus.ExponentialBuckets(0.0005, 2, 20), // 0.5ms ~ 262s Buckets: prometheus.ExponentialBuckets(0.0005, 2, 20), // 0.5ms ~ 262s
ConstLabels: constLabels,
}) })
TiKVStatusDuration = prometheus.NewHistogramVec( TiKVStatusDuration = prometheus.NewHistogramVec(
prometheus.HistogramOpts{ prometheus.HistogramOpts{
Namespace: namespace, Namespace: namespace,
Subsystem: subsystem, Subsystem: subsystem,
Name: "kv_status_api_duration", Name: "kv_status_api_duration",
Help: "duration for kv status api.", Help: "duration for kv status api.",
Buckets: prometheus.ExponentialBuckets(0.0005, 2, 20), // 0.5ms ~ 262s Buckets: prometheus.ExponentialBuckets(0.0005, 2, 20), // 0.5ms ~ 262s
ConstLabels: constLabels,
}, []string{"store"}) }, []string{"store"})
TiKVStatusCounter = prometheus.NewCounterVec( TiKVStatusCounter = prometheus.NewCounterVec(
prometheus.CounterOpts{ prometheus.CounterOpts{
Namespace: namespace, Namespace: namespace,
Subsystem: subsystem, Subsystem: subsystem,
Name: "kv_status_api_count", Name: "kv_status_api_count",
Help: "Counter of access kv status api.", Help: "Counter of access kv status api.",
ConstLabels: constLabels,
}, []string{LblResult}) }, []string{LblResult})
TiKVBatchWaitDuration = prometheus.NewHistogram( TiKVBatchWaitDuration = prometheus.NewHistogram(
prometheus.HistogramOpts{ prometheus.HistogramOpts{
Namespace: namespace, Namespace: namespace,
Subsystem: subsystem, Subsystem: subsystem,
Name: "batch_wait_duration", Name: "batch_wait_duration",
Buckets: prometheus.ExponentialBuckets(1, 2, 34), // 1ns ~ 8s Buckets: prometheus.ExponentialBuckets(1, 2, 34), // 1ns ~ 8s
Help: "batch wait duration", Help: "batch wait duration",
ConstLabels: constLabels,
}) })
TiKVBatchSendLatency = prometheus.NewHistogram( TiKVBatchSendLatency = prometheus.NewHistogram(
prometheus.HistogramOpts{ prometheus.HistogramOpts{
Namespace: namespace, Namespace: namespace,
Subsystem: subsystem, Subsystem: subsystem,
Name: "batch_send_latency", Name: "batch_send_latency",
Buckets: prometheus.ExponentialBuckets(1, 2, 34), // 1ns ~ 8s Buckets: prometheus.ExponentialBuckets(1, 2, 34), // 1ns ~ 8s
Help: "batch send latency", Help: "batch send latency",
ConstLabels: constLabels,
}) })
TiKVBatchRecvLatency = prometheus.NewHistogramVec( TiKVBatchRecvLatency = prometheus.NewHistogramVec(
prometheus.HistogramOpts{ prometheus.HistogramOpts{
Namespace: namespace, Namespace: namespace,
Subsystem: subsystem, Subsystem: subsystem,
Name: "batch_recv_latency", Name: "batch_recv_latency",
Buckets: prometheus.ExponentialBuckets(1000, 2, 34), // 1us ~ 8000s Buckets: prometheus.ExponentialBuckets(1000, 2, 34), // 1us ~ 8000s
Help: "batch recv latency", Help: "batch recv latency",
ConstLabels: constLabels,
}, []string{LblResult}) }, []string{LblResult})
TiKVBatchWaitOverLoad = prometheus.NewCounter( TiKVBatchWaitOverLoad = prometheus.NewCounter(
prometheus.CounterOpts{ prometheus.CounterOpts{
Namespace: namespace, Namespace: namespace,
Subsystem: subsystem, Subsystem: subsystem,
Name: "batch_wait_overload", Name: "batch_wait_overload",
Help: "event of tikv transport layer overload", Help: "event of tikv transport layer overload",
ConstLabels: constLabels,
}) })
TiKVBatchPendingRequests = prometheus.NewHistogramVec( TiKVBatchPendingRequests = prometheus.NewHistogramVec(
prometheus.HistogramOpts{ prometheus.HistogramOpts{
Namespace: namespace, Namespace: namespace,
Subsystem: subsystem, Subsystem: subsystem,
Name: "batch_pending_requests", Name: "batch_pending_requests",
Buckets: prometheus.ExponentialBuckets(1, 2, 11), // 1 ~ 1024 Buckets: prometheus.ExponentialBuckets(1, 2, 11), // 1 ~ 1024
Help: "number of requests pending in the batch channel", Help: "number of requests pending in the batch channel",
ConstLabels: constLabels,
}, []string{"store"}) }, []string{"store"})
TiKVBatchRequests = prometheus.NewHistogramVec( TiKVBatchRequests = prometheus.NewHistogramVec(
prometheus.HistogramOpts{ prometheus.HistogramOpts{
Namespace: namespace, Namespace: namespace,
Subsystem: subsystem, Subsystem: subsystem,
Name: "batch_requests", Name: "batch_requests",
Buckets: prometheus.ExponentialBuckets(1, 2, 11), // 1 ~ 1024 Buckets: prometheus.ExponentialBuckets(1, 2, 11), // 1 ~ 1024
Help: "number of requests in one batch", Help: "number of requests in one batch",
ConstLabels: constLabels,
}, []string{"store"}) }, []string{"store"})
TiKVBatchClientUnavailable = prometheus.NewHistogram( TiKVBatchClientUnavailable = prometheus.NewHistogram(
prometheus.HistogramOpts{ prometheus.HistogramOpts{
Namespace: namespace, Namespace: namespace,
Subsystem: subsystem, Subsystem: subsystem,
Name: "batch_client_unavailable_seconds", Name: "batch_client_unavailable_seconds",
Buckets: prometheus.ExponentialBuckets(0.001, 2, 28), // 1ms ~ 1.5days Buckets: prometheus.ExponentialBuckets(0.001, 2, 28), // 1ms ~ 1.5days
Help: "batch client unavailable", Help: "batch client unavailable",
ConstLabels: constLabels,
}) })
TiKVBatchClientWaitEstablish = prometheus.NewHistogram( TiKVBatchClientWaitEstablish = prometheus.NewHistogram(
prometheus.HistogramOpts{ prometheus.HistogramOpts{
Namespace: namespace, Namespace: namespace,
Subsystem: subsystem, Subsystem: subsystem,
Name: "batch_client_wait_connection_establish", Name: "batch_client_wait_connection_establish",
Buckets: prometheus.ExponentialBuckets(0.001, 2, 28), // 1ms ~ 1.5days Buckets: prometheus.ExponentialBuckets(0.001, 2, 28), // 1ms ~ 1.5days
Help: "batch client wait new connection establish", Help: "batch client wait new connection establish",
ConstLabels: constLabels,
}) })
TiKVBatchClientRecycle = prometheus.NewHistogram( TiKVBatchClientRecycle = prometheus.NewHistogram(
prometheus.HistogramOpts{ prometheus.HistogramOpts{
Namespace: namespace, Namespace: namespace,
Subsystem: subsystem, Subsystem: subsystem,
Name: "batch_client_reset", Name: "batch_client_reset",
Buckets: prometheus.ExponentialBuckets(0.001, 2, 28), // 1ms ~ 1.5days Buckets: prometheus.ExponentialBuckets(0.001, 2, 28), // 1ms ~ 1.5days
Help: "batch client recycle connection and reconnect duration", Help: "batch client recycle connection and reconnect duration",
ConstLabels: constLabels,
}) })
TiKVRangeTaskStats = prometheus.NewGaugeVec( TiKVRangeTaskStats = prometheus.NewGaugeVec(
prometheus.GaugeOpts{ prometheus.GaugeOpts{
Namespace: namespace, Namespace: namespace,
Subsystem: subsystem, Subsystem: subsystem,
Name: "range_task_stats", Name: "range_task_stats",
Help: "stat of range tasks", Help: "stat of range tasks",
ConstLabels: constLabels,
}, []string{LblType, LblResult}) }, []string{LblType, LblResult})
TiKVRangeTaskPushDuration = prometheus.NewHistogramVec( TiKVRangeTaskPushDuration = prometheus.NewHistogramVec(
prometheus.HistogramOpts{ prometheus.HistogramOpts{
Namespace: namespace, Namespace: namespace,
Subsystem: subsystem, Subsystem: subsystem,
Name: "range_task_push_duration", Name: "range_task_push_duration",
Buckets: prometheus.ExponentialBuckets(0.001, 2, 20), // 1ms ~ 524s Buckets: prometheus.ExponentialBuckets(0.001, 2, 20), // 1ms ~ 524s
Help: "duration to push sub tasks to range task workers", Help: "duration to push sub tasks to range task workers",
ConstLabels: constLabels,
}, []string{LblType}) }, []string{LblType})
TiKVTokenWaitDuration = prometheus.NewHistogram( TiKVTokenWaitDuration = prometheus.NewHistogram(
prometheus.HistogramOpts{ prometheus.HistogramOpts{
Namespace: namespace, Namespace: namespace,
Subsystem: subsystem, Subsystem: subsystem,
Name: "batch_executor_token_wait_duration", Name: "batch_executor_token_wait_duration",
Buckets: prometheus.ExponentialBuckets(1, 2, 34), // 1ns ~ 8s Buckets: prometheus.ExponentialBuckets(1, 2, 34), // 1ns ~ 8s
Help: "tidb txn token wait duration to process batches", Help: "tidb txn token wait duration to process batches",
ConstLabels: constLabels,
}) })
TiKVTxnHeartBeatHistogram = prometheus.NewHistogramVec( TiKVTxnHeartBeatHistogram = prometheus.NewHistogramVec(
prometheus.HistogramOpts{ prometheus.HistogramOpts{
Namespace: namespace, Namespace: namespace,
Subsystem: subsystem, Subsystem: subsystem,
Name: "txn_heart_beat", Name: "txn_heart_beat",
Help: "Bucketed histogram of the txn_heartbeat request duration.", Help: "Bucketed histogram of the txn_heartbeat request duration.",
Buckets: prometheus.ExponentialBuckets(0.001, 2, 20), // 1ms ~ 524s ConstLabels: constLabels,
Buckets: prometheus.ExponentialBuckets(0.001, 2, 20), // 1ms ~ 524s
}, []string{LblType}) }, []string{LblType})
TiKVTTLManagerHistogram = prometheus.NewHistogram( TiKVTTLManagerHistogram = prometheus.NewHistogram(
prometheus.HistogramOpts{ prometheus.HistogramOpts{
Namespace: namespace, Namespace: namespace,
Subsystem: subsystem, Subsystem: subsystem,
Name: "txn_ttl_manager", Name: "txn_ttl_manager",
Help: "Bucketed histogram of the txn ttl manager lifetime duration.", Help: "Bucketed histogram of the txn ttl manager lifetime duration.",
Buckets: prometheus.ExponentialBuckets(1, 2, 20), // 1s ~ 524288s ConstLabels: constLabels,
Buckets: prometheus.ExponentialBuckets(1, 2, 20), // 1s ~ 524288s
}) })
TiKVPessimisticLockKeysDuration = prometheus.NewHistogram( TiKVPessimisticLockKeysDuration = prometheus.NewHistogram(
prometheus.HistogramOpts{ prometheus.HistogramOpts{
Namespace: namespace, Namespace: namespace,
Subsystem: subsystem, Subsystem: subsystem,
Name: "pessimistic_lock_keys_duration", Name: "pessimistic_lock_keys_duration",
Buckets: prometheus.ExponentialBuckets(0.001, 2, 24), // 1ms ~ 8389s Buckets: prometheus.ExponentialBuckets(0.001, 2, 24), // 1ms ~ 8389s
Help: "tidb txn pessimistic lock keys duration", Help: "tidb txn pessimistic lock keys duration",
ConstLabels: constLabels,
}) })
TiKVTTLLifeTimeReachCounter = prometheus.NewCounter( TiKVTTLLifeTimeReachCounter = prometheus.NewCounter(
prometheus.CounterOpts{ prometheus.CounterOpts{
Namespace: namespace, Namespace: namespace,
Subsystem: subsystem, Subsystem: subsystem,
Name: "ttl_lifetime_reach_total", Name: "ttl_lifetime_reach_total",
Help: "Counter of ttlManager live too long.", Help: "Counter of ttlManager live too long.",
ConstLabels: constLabels,
}) })
TiKVNoAvailableConnectionCounter = prometheus.NewCounter( TiKVNoAvailableConnectionCounter = prometheus.NewCounter(
prometheus.CounterOpts{ prometheus.CounterOpts{
Namespace: namespace, Namespace: namespace,
Subsystem: subsystem, Subsystem: subsystem,
Name: "batch_client_no_available_connection_total", Name: "batch_client_no_available_connection_total",
Help: "Counter of no available batch client.", Help: "Counter of no available batch client.",
ConstLabels: constLabels,
}) })
TiKVTwoPCTxnCounter = prometheus.NewCounterVec( TiKVTwoPCTxnCounter = prometheus.NewCounterVec(
prometheus.CounterOpts{ prometheus.CounterOpts{
Namespace: namespace, Namespace: namespace,
Subsystem: subsystem, Subsystem: subsystem,
Name: "commit_txn_counter", Name: "commit_txn_counter",
Help: "Counter of 2PC transactions.", Help: "Counter of 2PC transactions.",
ConstLabels: constLabels,
}, []string{LblType}) }, []string{LblType})
TiKVAsyncCommitTxnCounter = prometheus.NewCounterVec( TiKVAsyncCommitTxnCounter = prometheus.NewCounterVec(
prometheus.CounterOpts{ prometheus.CounterOpts{
Namespace: namespace, Namespace: namespace,
Subsystem: subsystem, Subsystem: subsystem,
Name: "async_commit_txn_counter", Name: "async_commit_txn_counter",
Help: "Counter of async commit transactions.", Help: "Counter of async commit transactions.",
ConstLabels: constLabels,
}, []string{LblType}) }, []string{LblType})
TiKVOnePCTxnCounter = prometheus.NewCounterVec( TiKVOnePCTxnCounter = prometheus.NewCounterVec(
prometheus.CounterOpts{ prometheus.CounterOpts{
Namespace: namespace, Namespace: namespace,
Subsystem: subsystem, Subsystem: subsystem,
Name: "one_pc_txn_counter", Name: "one_pc_txn_counter",
Help: "Counter of 1PC transactions.", Help: "Counter of 1PC transactions.",
ConstLabels: constLabels,
}, []string{LblType}) }, []string{LblType})
TiKVStoreLimitErrorCounter = prometheus.NewCounterVec( TiKVStoreLimitErrorCounter = prometheus.NewCounterVec(
prometheus.CounterOpts{ prometheus.CounterOpts{
Namespace: namespace, Namespace: namespace,
Subsystem: subsystem, Subsystem: subsystem,
Name: "get_store_limit_token_error", Name: "get_store_limit_token_error",
Help: "store token is up to the limit, probably because one of the stores is the hotspot or unavailable", Help: "store token is up to the limit, probably because one of the stores is the hotspot or unavailable",
ConstLabels: constLabels,
}, []string{LblAddress, LblStore}) }, []string{LblAddress, LblStore})
TiKVGRPCConnTransientFailureCounter = prometheus.NewCounterVec( TiKVGRPCConnTransientFailureCounter = prometheus.NewCounterVec(
prometheus.CounterOpts{ prometheus.CounterOpts{
Namespace: namespace, Namespace: namespace,
Subsystem: subsystem, Subsystem: subsystem,
Name: "connection_transient_failure_count", Name: "connection_transient_failure_count",
Help: "Counter of gRPC connection transient failure", Help: "Counter of gRPC connection transient failure",
ConstLabels: constLabels,
}, []string{LblAddress, LblStore}) }, []string{LblAddress, LblStore})
TiKVPanicCounter = prometheus.NewCounterVec( TiKVPanicCounter = prometheus.NewCounterVec(
prometheus.CounterOpts{ prometheus.CounterOpts{
Namespace: namespace, Namespace: namespace,
Subsystem: subsystem, Subsystem: subsystem,
Name: "panic_total", Name: "panic_total",
Help: "Counter of panic.", Help: "Counter of panic.",
ConstLabels: constLabels,
}, []string{LblType}) }, []string{LblType})
TiKVForwardRequestCounter = prometheus.NewCounterVec( TiKVForwardRequestCounter = prometheus.NewCounterVec(
prometheus.CounterOpts{ prometheus.CounterOpts{
Namespace: namespace, Namespace: namespace,
Subsystem: subsystem, Subsystem: subsystem,
Name: "forward_request_counter", Name: "forward_request_counter",
Help: "Counter of tikv request being forwarded through another node", Help: "Counter of tikv request being forwarded through another node",
ConstLabels: constLabels,
}, []string{LblFromStore, LblToStore, LblType, LblResult}) }, []string{LblFromStore, LblToStore, LblType, LblResult})
TiKVTSFutureWaitDuration = prometheus.NewHistogram( TiKVTSFutureWaitDuration = prometheus.NewHistogram(
prometheus.HistogramOpts{ prometheus.HistogramOpts{
Namespace: namespace, Namespace: namespace,
Subsystem: subsystem, Subsystem: subsystem,
Name: "ts_future_wait_seconds", Name: "ts_future_wait_seconds",
Help: "Bucketed histogram of seconds cost for waiting timestamp future.", Help: "Bucketed histogram of seconds cost for waiting timestamp future.",
Buckets: prometheus.ExponentialBuckets(0.000005, 2, 30), // 5us ~ 2560s ConstLabels: constLabels,
Buckets: prometheus.ExponentialBuckets(0.000005, 2, 30), // 5us ~ 2560s
}) })
TiKVSafeTSUpdateCounter = prometheus.NewCounterVec( TiKVSafeTSUpdateCounter = prometheus.NewCounterVec(
prometheus.CounterOpts{ prometheus.CounterOpts{
Namespace: namespace, Namespace: namespace,
Subsystem: subsystem, Subsystem: subsystem,
Name: "safets_update_counter", Name: "safets_update_counter",
Help: "Counter of tikv safe_ts being updated.", Help: "Counter of tikv safe_ts being updated.",
ConstLabels: constLabels,
}, []string{LblResult, LblStore}) }, []string{LblResult, LblStore})
TiKVMinSafeTSGapSeconds = prometheus.NewGaugeVec( TiKVMinSafeTSGapSeconds = prometheus.NewGaugeVec(
prometheus.GaugeOpts{ prometheus.GaugeOpts{
Namespace: namespace, Namespace: namespace,
Subsystem: subsystem, Subsystem: subsystem,
Name: "min_safets_gap_seconds", Name: "min_safets_gap_seconds",
Help: "The minimal (non-zero) SafeTS gap for each store.", Help: "The minimal (non-zero) SafeTS gap for each store.",
ConstLabels: constLabels,
}, []string{LblStore}) }, []string{LblStore})
TiKVReplicaSelectorFailureCounter = prometheus.NewCounterVec( TiKVReplicaSelectorFailureCounter = prometheus.NewCounterVec(
prometheus.CounterOpts{ prometheus.CounterOpts{
Namespace: namespace, Namespace: namespace,
Subsystem: subsystem, Subsystem: subsystem,
Name: "replica_selector_failure_counter", Name: "replica_selector_failure_counter",
Help: "Counter of the reason why the replica selector cannot yield a potential leader.", Help: "Counter of the reason why the replica selector cannot yield a potential leader.",
ConstLabels: constLabels,
}, []string{LblType}) }, []string{LblType})
TiKVRequestRetryTimesHistogram = prometheus.NewHistogram( TiKVRequestRetryTimesHistogram = prometheus.NewHistogram(
prometheus.HistogramOpts{ prometheus.HistogramOpts{
Namespace: namespace, Namespace: namespace,
Subsystem: subsystem, Subsystem: subsystem,
Name: "request_retry_times", Name: "request_retry_times",
Help: "Bucketed histogram of how many times a region request retries.", Help: "Bucketed histogram of how many times a region request retries.",
Buckets: []float64{1, 2, 3, 4, 8, 16, 32, 64, 128, 256}, ConstLabels: constLabels,
Buckets: []float64{1, 2, 3, 4, 8, 16, 32, 64, 128, 256},
}) })
TiKVTxnCommitBackoffSeconds = prometheus.NewHistogram( TiKVTxnCommitBackoffSeconds = prometheus.NewHistogram(
prometheus.HistogramOpts{ prometheus.HistogramOpts{
Namespace: namespace, Namespace: namespace,
Subsystem: subsystem, Subsystem: subsystem,
Name: "txn_commit_backoff_seconds", Name: "txn_commit_backoff_seconds",
Help: "Bucketed histogram of the total backoff duration in committing a transaction.", Help: "Bucketed histogram of the total backoff duration in committing a transaction.",
Buckets: prometheus.ExponentialBuckets(0.001, 2, 22), // 1ms ~ 2097s ConstLabels: constLabels,
Buckets: prometheus.ExponentialBuckets(0.001, 2, 22), // 1ms ~ 2097s
}) })
TiKVTxnCommitBackoffCount = prometheus.NewHistogram( TiKVTxnCommitBackoffCount = prometheus.NewHistogram(
prometheus.HistogramOpts{ prometheus.HistogramOpts{
Namespace: namespace, Namespace: namespace,
Subsystem: subsystem, Subsystem: subsystem,
Name: "txn_commit_backoff_count", Name: "txn_commit_backoff_count",
Help: "Bucketed histogram of the backoff count in committing a transaction.", Help: "Bucketed histogram of the backoff count in committing a transaction.",
Buckets: prometheus.ExponentialBuckets(1, 2, 12), // 1 ~ 2048 ConstLabels: constLabels,
Buckets: prometheus.ExponentialBuckets(1, 2, 12), // 1 ~ 2048
}) })
// TiKVSmallReadDuration uses to collect small request read duration. // TiKVSmallReadDuration uses to collect small request read duration.
TiKVSmallReadDuration = prometheus.NewHistogram( TiKVSmallReadDuration = prometheus.NewHistogram(
prometheus.HistogramOpts{ prometheus.HistogramOpts{
Namespace: namespace, Namespace: namespace,
Subsystem: "sli", // Always use "sli" to make it compatible with TiDB. Subsystem: "sli", // Always use "sli" to make it compatible with TiDB.
Name: "tikv_small_read_duration", Name: "tikv_small_read_duration",
Help: "Read time of TiKV small read.", Help: "Read time of TiKV small read.",
Buckets: prometheus.ExponentialBuckets(0.0005, 2, 28), // 0.5ms ~ 74h ConstLabels: constLabels,
Buckets: prometheus.ExponentialBuckets(0.0005, 2, 28), // 0.5ms ~ 74h
}) })
TiKVReadThroughput = prometheus.NewHistogram( TiKVReadThroughput = prometheus.NewHistogram(
prometheus.HistogramOpts{ prometheus.HistogramOpts{
Namespace: namespace, Namespace: namespace,
Subsystem: "sli", Subsystem: "sli",
Name: "tikv_read_throughput", Name: "tikv_read_throughput",
Help: "Read throughput of TiKV read in Bytes/s.", Help: "Read throughput of TiKV read in Bytes/s.",
Buckets: prometheus.ExponentialBuckets(1024, 2, 13), // 1MB/s ~ 4GB/s ConstLabels: constLabels,
Buckets: prometheus.ExponentialBuckets(1024, 2, 13), // 1MB/s ~ 4GB/s
}) })
TiKVUnsafeDestroyRangeFailuresCounterVec = prometheus.NewCounterVec( TiKVUnsafeDestroyRangeFailuresCounterVec = prometheus.NewCounterVec(
prometheus.CounterOpts{ prometheus.CounterOpts{
Namespace: namespace, Namespace: namespace,
Subsystem: subsystem, Subsystem: subsystem,
Name: "gc_unsafe_destroy_range_failures", Name: "gc_unsafe_destroy_range_failures",
Help: "Counter of unsafe destroyrange failures", Help: "Counter of unsafe destroyrange failures",
ConstLabels: constLabels,
}, []string{LblType}) }, []string{LblType})
TiKVPrewriteAssertionUsageCounter = prometheus.NewCounterVec( TiKVPrewriteAssertionUsageCounter = prometheus.NewCounterVec(
prometheus.CounterOpts{ prometheus.CounterOpts{
Namespace: namespace, Namespace: namespace,
Subsystem: subsystem, Subsystem: subsystem,
Name: "prewrite_assertion_count", Name: "prewrite_assertion_count",
Help: "Counter of assertions used in prewrite requests", Help: "Counter of assertions used in prewrite requests",
ConstLabels: constLabels,
}, []string{LblType}) }, []string{LblType})
TiKVGrpcConnectionState = prometheus.NewGaugeVec( TiKVGrpcConnectionState = prometheus.NewGaugeVec(
prometheus.GaugeOpts{ prometheus.GaugeOpts{
Namespace: namespace, Namespace: namespace,
Subsystem: subsystem, Subsystem: subsystem,
Name: "grpc_connection_state", Name: "grpc_connection_state",
Help: "State of gRPC connection", Help: "State of gRPC connection",
ConstLabels: constLabels,
}, []string{"connection_id", "store_ip", "grpc_state"}) }, []string{"connection_id", "store_ip", "grpc_state"})
TiKVAggressiveLockedKeysCounter = prometheus.NewCounterVec( TiKVAggressiveLockedKeysCounter = prometheus.NewCounterVec(
prometheus.CounterOpts{ prometheus.CounterOpts{
Namespace: namespace, Namespace: namespace,
Subsystem: subsystem, Subsystem: subsystem,
Name: "aggressive_locking_count", Name: "aggressive_locking_count",
Help: "Counter of keys locked in aggressive locking mode", Help: "Counter of keys locked in aggressive locking mode",
ConstLabels: constLabels,
}, []string{LblType}) }, []string{LblType})
TiKVStoreSlowScoreGauge = prometheus.NewGaugeVec( TiKVStoreSlowScoreGauge = prometheus.NewGaugeVec(
prometheus.GaugeOpts{ prometheus.GaugeOpts{
Namespace: namespace, Namespace: namespace,
Subsystem: subsystem, Subsystem: subsystem,
Name: "store_slow_score", Name: "store_slow_score",
Help: "Slow scores of each tikv node based on RPC timecosts", Help: "Slow scores of each tikv node based on RPC timecosts",
ConstLabels: constLabels,
}, []string{LblStore}) }, []string{LblStore})
TiKVPreferLeaderFlowsGauge = prometheus.NewGaugeVec( TiKVPreferLeaderFlowsGauge = prometheus.NewGaugeVec(
prometheus.GaugeOpts{ prometheus.GaugeOpts{
Namespace: namespace, Namespace: namespace,
Subsystem: subsystem, Subsystem: subsystem,
Name: "prefer_leader_flows_gauge", Name: "prefer_leader_flows_gauge",
Help: "Counter of flows under PreferLeader mode.", Help: "Counter of flows under PreferLeader mode.",
ConstLabels: constLabels,
}, []string{LblType, LblStore}) }, []string{LblType, LblStore})
TiKVStaleReadSizeSummary = prometheus.NewSummaryVec( TiKVStaleReadSizeSummary = prometheus.NewSummaryVec(
prometheus.SummaryOpts{ prometheus.SummaryOpts{
Namespace: namespace, Namespace: namespace,
Subsystem: subsystem, Subsystem: subsystem,
Name: "stale_read_bytes", Name: "stale_read_bytes",
Help: "Size of stale read.", Help: "Size of stale read.",
ConstLabels: constLabels,
}, []string{LblResult, LblDirection}) }, []string{LblResult, LblDirection})
initShortcuts() initShortcuts()
} }
func init() { func init() {
initMetrics("tikv", "client_go") initMetrics("tikv", "client_go", nil)
} }
// InitMetrics initializes metrics variables with given namespace and subsystem name. // InitMetrics initializes metrics variables with given namespace and subsystem name.
func InitMetrics(namespace, subsystem string) { func InitMetrics(namespace, subsystem string) {
initMetrics(namespace, subsystem) initMetrics(namespace, subsystem, nil)
}
// InitMetricsWithConstLabels initializes metrics variables with given namespace, subsystem name and const labels.
func InitMetricsWithConstLabels(namespace, subsystem string, constLabels prometheus.Labels) {
initMetrics(namespace, subsystem, constLabels)
} }
// RegisterMetrics registers all metrics variables. // RegisterMetrics registers all metrics variables.