# these rules synced manually from https://github.com/etcd-io/etcd/blob/release-3.5/contrib/mixin/mixin.libsonnet
groups:
- name: etcd
  rules:
  - alert: etcdMembersDown
    annotations:
      description: 'etcd cluster "{{ $labels.job }}": members are down ({{ $value }}).'
      summary: etcd cluster members are down.
    expr: |
      max without (endpoint) (
        sum without (instance) (up{job=~".*etcd.*"} == bool 0)
      or
        count without (To) (
          sum without (instance) (rate(etcd_network_peer_sent_failures_total{job=~".*etcd.*"}[120s])) > 0.01
        )
      )
      > 0
    for: 10m
    labels:
      severity: critical
  - alert: etcdInsufficientMembers
    annotations:
      description: 'etcd cluster "{{ $labels.job }}": insufficient members ({{ $value }}).'
      summary: etcd cluster has insufficient number of members.
    expr: |
      sum(up{job=~".*etcd.*"} == bool 1) without (instance) < ((count(up{job=~".*etcd.*"}) without (instance) + 1) / 2)
    for: 3m
    labels:
      severity: critical
  - alert: etcdNoLeader
    annotations:
      description: 'etcd cluster "{{ $labels.job }}": member {{ $labels.instance }} has no leader.'
      summary: etcd cluster has no leader.
    expr: |
      etcd_server_has_leader{job=~".*etcd.*"} == 0
    for: 1m
    labels:
      severity: critical
  - alert: etcdHighNumberOfLeaderChanges
    annotations:
      description: 'etcd cluster "{{ $labels.job }}": {{ $value }} leader changes within the last 15 minutes. Frequent elections may be a sign of insufficient resources, high network latency, or disruptions by other components and should be investigated.'
      summary: etcd cluster has high number of leader changes.
    expr: |
      increase((max without (instance) (etcd_server_leader_changes_seen_total{job=~".*etcd.*"}) or 0*absent(etcd_server_leader_changes_seen_total{job=~".*etcd.*"}))[15m:1m]) >= 4
    for: 5m
    labels:
      severity: warning
  - alert: etcdHighNumberOfFailedGRPCRequests
    annotations:
      description: 'etcd cluster "{{ $labels.job }}": {{ $value }}% of requests for {{ $labels.grpc_method }} failed on etcd instance {{ $labels.instance }}.'
      summary: etcd cluster has high number of failed grpc requests.
    expr: |
      100 * sum(rate(grpc_server_handled_total{job=~".*etcd.*", grpc_code=~"Unknown|FailedPrecondition|ResourceExhausted|Internal|Unavailable|DataLoss|DeadlineExceeded"}[5m])) without (grpc_type, grpc_code)
        /
      sum(rate(grpc_server_handled_total{job=~".*etcd.*"}[5m])) without (grpc_type, grpc_code)
        > 1
    for: 10m
    labels:
      severity: warning
  - alert: etcdHighNumberOfFailedGRPCRequests
    annotations:
      description: 'etcd cluster "{{ $labels.job }}": {{ $value }}% of requests for {{ $labels.grpc_method }} failed on etcd instance {{ $labels.instance }}.'
      summary: etcd cluster has high number of failed grpc requests.
    expr: |
      100 * sum(rate(grpc_server_handled_total{job=~".*etcd.*", grpc_code=~"Unknown|FailedPrecondition|ResourceExhausted|Internal|Unavailable|DataLoss|DeadlineExceeded"}[5m])) without (grpc_type, grpc_code)
        /
      sum(rate(grpc_server_handled_total{job=~".*etcd.*"}[5m])) without (grpc_type, grpc_code)
        > 5
    for: 5m
    labels:
      severity: critical
  - alert: etcdGRPCRequestsSlow
    annotations:
      description: 'etcd cluster "{{ $labels.job }}": 99th percentile of gRPC requests is {{ $value }}s on etcd instance {{ $labels.instance }} for {{ $labels.grpc_method }} method.'
      summary: etcd grpc requests are slow
    expr: |
      histogram_quantile(0.99, sum(rate(grpc_server_handling_seconds_bucket{job=~".*etcd.*", grpc_method!="Defragment", grpc_type="unary"}[5m])) without(grpc_type))
      > 0.15
    for: 10m
    labels:
      severity: critical
  - alert: etcdMemberCommunicationSlow
    annotations:
      description: 'etcd cluster "{{ $labels.job }}": member communication with {{ $labels.To }} is taking {{ $value }}s on etcd instance {{ $labels.instance }}.'
      summary: etcd cluster member communication is slow.
    expr: |
      histogram_quantile(0.99, rate(etcd_network_peer_round_trip_time_seconds_bucket{job=~".*etcd.*"}[5m]))
      > 0.15
    for: 10m
    labels:
      severity: warning
  - alert: etcdHighNumberOfFailedProposals
    annotations:
      description: 'etcd cluster "{{ $labels.job }}": {{ $value }} proposal failures within the last 30 minutes on etcd instance {{ $labels.instance }}.'
      summary: etcd cluster has high number of proposal failures.
    expr: |
      rate(etcd_server_proposals_failed_total{job=~".*etcd.*"}[15m]) > 5
    for: 15m
    labels:
      severity: warning
  - alert: etcdHighFsyncDurations
    annotations:
      description: 'etcd cluster "{{ $labels.job }}": 99th percentile fsync durations are {{ $value }}s on etcd instance {{ $labels.instance }}.'
      summary: etcd cluster 99th percentile fsync durations are too high.
    expr: |
      histogram_quantile(0.99, rate(etcd_disk_wal_fsync_duration_seconds_bucket{job=~".*etcd.*"}[5m]))
      > 0.5
    for: 10m
    labels:
      severity: warning
  - alert: etcdHighFsyncDurations
    annotations:
      description: 'etcd cluster "{{ $labels.job }}": 99th percentile fsync durations are {{ $value }}s on etcd instance {{ $labels.instance }}.'
      summary: etcd cluster 99th percentile fsync durations are too high.
    expr: |
      histogram_quantile(0.99, rate(etcd_disk_wal_fsync_duration_seconds_bucket{job=~".*etcd.*"}[5m]))
      > 1
    for: 10m
    labels:
      severity: critical
  - alert: etcdHighCommitDurations
    annotations:
      description: 'etcd cluster "{{ $labels.job }}": 99th percentile commit durations {{ $value }}s on etcd instance {{ $labels.instance }}.'
      summary: etcd cluster 99th percentile commit durations are too high.
    expr: |
      histogram_quantile(0.99, rate(etcd_disk_backend_commit_duration_seconds_bucket{job=~".*etcd.*"}[5m]))
      > 0.25
    for: 10m
    labels:
      severity: warning
  - alert: etcdBackendQuotaLowSpace
    annotations:
      description: 'etcd cluster "{{ $labels.job }}": database size exceeds the defined quota on etcd instance {{ $labels.instance }}, please defrag or increase the quota as the writes to etcd will be disabled when it is full.'
      summary: etcd cluster database is running full.
    expr: |
      (etcd_mvcc_db_total_size_in_bytes/etcd_server_quota_backend_bytes)*100 > 95
    for: 10m
    labels:
      severity: critical
  - alert: etcdExcessiveDatabaseGrowth
    annotations:
      description: 'etcd cluster "{{ $labels.job }}": Predicting running out of disk space in the next four hours, based on write observations within the past four hours on etcd instance {{ $labels.instance }}, please check as it might be disruptive.'
      summary: etcd cluster database growing very fast.
    expr: |
      predict_linear(etcd_mvcc_db_total_size_in_bytes[4h], 4*60*60) > etcd_server_quota_backend_bytes
    for: 10m
    labels:
      severity: warning
