| 1 |
rule_files: |
| 2 |
- mixin.yaml |
| 3 |
|
| 4 |
evaluation_interval: 1m |
| 5 |
|
| 6 |
tests: |
| 7 |
- interval: 1m |
| 8 |
input_series: |
| 9 |
- series: 'up{job="etcd",instance="10.10.10.0"}' |
| 10 |
values: '1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0' |
| 11 |
- series: 'up{job="etcd",instance="10.10.10.1"}' |
| 12 |
values: '1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0' |
| 13 |
- series: 'up{job="etcd",instance="10.10.10.2"}' |
| 14 |
values: '1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0' |
| 15 |
alert_rule_test: |
| 16 |
- eval_time: 3m |
| 17 |
alertname: etcdInsufficientMembers |
| 18 |
- eval_time: 5m |
| 19 |
alertname: etcdInsufficientMembers |
| 20 |
- eval_time: 12m |
| 21 |
alertname: etcdMembersDown |
| 22 |
- eval_time: 14m |
| 23 |
alertname: etcdMembersDown |
| 24 |
exp_alerts: |
| 25 |
- exp_labels: |
| 26 |
job: etcd |
| 27 |
severity: critical |
| 28 |
exp_annotations: |
| 29 |
description: 'etcd cluster "etcd": members are down (3).' |
| 30 |
summary: 'etcd cluster members are down.' |
| 31 |
- eval_time: 7m |
| 32 |
alertname: etcdInsufficientMembers |
| 33 |
- eval_time: 11m |
| 34 |
alertname: etcdInsufficientMembers |
| 35 |
exp_alerts: |
| 36 |
- exp_labels: |
| 37 |
job: etcd |
| 38 |
severity: critical |
| 39 |
exp_annotations: |
| 40 |
description: 'etcd cluster "etcd": insufficient members (1).' |
| 41 |
summary: 'etcd cluster has insufficient number of members.' |
| 42 |
- eval_time: 15m |
| 43 |
alertname: etcdInsufficientMembers |
| 44 |
exp_alerts: |
| 45 |
- exp_labels: |
| 46 |
job: etcd |
| 47 |
severity: critical |
| 48 |
exp_annotations: |
| 49 |
description: 'etcd cluster "etcd": insufficient members (0).' |
| 50 |
summary: 'etcd cluster has insufficient number of members.' |
| 51 |
|
| 52 |
- interval: 1m |
| 53 |
input_series: |
| 54 |
- series: 'up{job="etcd",instance="10.10.10.0"}' |
| 55 |
values: '1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0' |
| 56 |
- series: 'up{job="etcd",instance="10.10.10.1"}' |
| 57 |
values: '1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0' |
| 58 |
- series: 'up{job="etcd",instance="10.10.10.2"}' |
| 59 |
values: '1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0' |
| 60 |
alert_rule_test: |
| 61 |
- eval_time: 14m |
| 62 |
alertname: etcdMembersDown |
| 63 |
exp_alerts: |
| 64 |
- exp_labels: |
| 65 |
job: etcd |
| 66 |
severity: critical |
| 67 |
exp_annotations: |
| 68 |
description: 'etcd cluster "etcd": members are down (3).' |
| 69 |
summary: 'etcd cluster members are down.' |
| 70 |
|
| 71 |
- interval: 1m |
| 72 |
input_series: |
| 73 |
- series: 'up{job="etcd",instance="10.10.10.0"}' |
| 74 |
values: '1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0' |
| 75 |
- series: 'up{job="etcd",instance="10.10.10.1"}' |
| 76 |
values: '1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0' |
| 77 |
- series: 'etcd_network_peer_sent_failures_total{To="member-1",job="etcd",endpoint="test"}' |
| 78 |
values: '0 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18' |
| 79 |
alert_rule_test: |
| 80 |
- eval_time: 13m |
| 81 |
alertname: etcdMembersDown |
| 82 |
exp_alerts: |
| 83 |
- exp_labels: |
| 84 |
job: etcd |
| 85 |
severity: critical |
| 86 |
exp_annotations: |
| 87 |
description: 'etcd cluster "etcd": members are down (1).' |
| 88 |
summary: 'etcd cluster members are down.' |
| 89 |
- interval: 1m |
| 90 |
input_series: |
| 91 |
- series: 'etcd_server_leader_changes_seen_total{job="etcd",instance="10.10.10.0"}' |
| 92 |
values: '0 0 2 0 0 1 0 0 0 0 0 0 0 0 0 0' |
| 93 |
- series: 'etcd_server_leader_changes_seen_total{job="etcd",instance="10.10.10.1"}' |
| 94 |
values: '0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0' |
| 95 |
- series: 'etcd_server_leader_changes_seen_total{job="etcd",instance="10.10.10.2"}' |
| 96 |
values: '0 0 0 0 0 0 0 0' |
| 97 |
alert_rule_test: |
| 98 |
- eval_time: 10m |
| 99 |
alertname: etcdHighNumberOfLeaderChanges |
| 100 |
exp_alerts: |
| 101 |
- exp_labels: |
| 102 |
job: etcd |
| 103 |
severity: warning |
| 104 |
exp_annotations: |
| 105 |
description: 'etcd cluster "etcd": 4 leader changes within the last 15 minutes. Frequent elections may be a sign of insufficient resources, high network latency, or disruptions by other components and should be investigated.' |
| 106 |
summary: 'etcd cluster has high number of leader changes.' |
| 107 |
- interval: 1m |
| 108 |
input_series: |
| 109 |
- series: 'etcd_server_leader_changes_seen_total{job="etcd",instance="10.10.10.0"}' |
| 110 |
values: '0 0 2 0 0 0 0 0 0 0 0 0 0 0 0 0' |
| 111 |
- series: 'etcd_server_leader_changes_seen_total{job="etcd",instance="10.10.10.1"}' |
| 112 |
values: '0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0' |
| 113 |
- series: 'etcd_server_leader_changes_seen_total{job="etcd",instance="10.10.10.2"}' |
| 114 |
values: '0 0 0 0 0 0 0 0' |
| 115 |
alert_rule_test: |
| 116 |
- eval_time: 10m |
| 117 |
alertname: etcdHighNumberOfLeaderChanges |
| 118 |
exp_alerts: |
| 119 |
- interval: 1m |
| 120 |
input_series: |
| 121 |
- series: '((etcd_mvcc_db_total_size_in_bytes/etcd_server_quota_backend_bytes)*100){job="etcd",instance="10.10.10.0"}' |
| 122 |
values: '0 10 20 0 0 10 0 0 30 0 0 0 0 0 0 0' |
| 123 |
- series: '((etcd_mvcc_db_total_size_in_bytes/etcd_server_quota_backend_bytes)*100){job="etcd",instance="10.10.10.1"}' |
| 124 |
values: '0 0 10 0 20 0 0 0 0 0 0 0 0 0 0 0' |
| 125 |
- series: '((etcd_mvcc_db_total_size_in_bytes/etcd_server_quota_backend_bytes)*100){job="etcd",instance="10.10.10.2"}' |
| 126 |
values: '0 0 0 0 0 0 0 0' |
| 127 |
alert_rule_test: |
| 128 |
- eval_time: 10m |
| 129 |
alertname: etcdExcessiveDatabaseGrowth |
| 130 |
exp_alerts: |
| 131 |
- exp_labels: |
| 132 |
job: etcd |
| 133 |
severity: warning |
| 134 |
exp_annotations: |
| 135 |
message: 'etcd cluster "etcd": Observed surge in etcd writes leading to 50% increase in database size over the past four hours, please check as it might be disruptive.' |