Debian Codegraph @

documentation/prometheus-mixin/alerts.libsonnet

raw
1 {
2 prometheusAlerts+:: {
3 groups+: [
4 {
5 name: 'prometheus',
6 rules: [
7 {
8 alert: 'PrometheusBadConfig',
9 expr: |||
10 # Without max_over_time, failed scrapes could create false negatives, see
11 # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details.
12 max_over_time(prometheus_config_last_reload_successful{%(prometheusSelector)s}[5m]) == 0
13 ||| % $._config,
14 'for': '10m',
15 labels: {
16 severity: 'critical',
17 },
18 annotations: {
19 summary: 'Failed Prometheus configuration reload.',
20 description: 'Prometheus %(prometheusName)s has failed to reload its configuration.' % $._config,
21 },
22 },
23 {
24 alert: 'PrometheusSDRefreshFailure',
25 expr: |||
26 increase(prometheus_sd_refresh_failures_total{%(prometheusSelector)s}[10m]) > 0
27 ||| % $._config,
28 'for': '20m',
29 labels: {
30 severity: 'warning',
31 },
32 annotations: {
33 summary: 'Failed Prometheus SD refresh.',
34 description: 'Prometheus %(prometheusName)s has failed to refresh SD with mechanism {{$labels.mechanism}}.' % $._config,
35 },
36 },
37 {
38 alert: 'PrometheusNotificationQueueRunningFull',
39 expr: |||
40 # Without min_over_time, failed scrapes could create false negatives, see
41 # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details.
42 (
43 predict_linear(prometheus_notifications_queue_length{%(prometheusSelector)s}[5m], 60 * 30)
44 >
45 min_over_time(prometheus_notifications_queue_capacity{%(prometheusSelector)s}[5m])
46 )
47 ||| % $._config,
48 'for': '15m',
49 labels: {
50 severity: 'warning',
51 },
52 annotations: {
53 summary: 'Prometheus alert notification queue predicted to run full in less than 30m.',
54 description: 'Alert notification queue of Prometheus %(prometheusName)s is running full.' % $._config,
55 },
56 },
57 {
58 alert: 'PrometheusErrorSendingAlertsToSomeAlertmanagers',
59 expr: |||
60 (
61 rate(prometheus_notifications_errors_total{%(prometheusSelector)s}[5m])
62 /
63 rate(prometheus_notifications_sent_total{%(prometheusSelector)s}[5m])
64 )
65 * 100
66 > 1
67 ||| % $._config,
68 'for': '15m',
69 labels: {
70 severity: 'warning',
71 },
72 annotations: {
73 summary: 'Prometheus has encountered more than 1% errors sending alerts to a specific Alertmanager.',
74 description: '{{ printf "%%.1f" $value }}%% errors while sending alerts from Prometheus %(prometheusName)s to Alertmanager {{$labels.alertmanager}}.' % $._config,
75 },
76 },
77 {
78 alert: 'PrometheusNotConnectedToAlertmanagers',
79 expr: |||
80 # Without max_over_time, failed scrapes could create false negatives, see
81 # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details.
82 max_over_time(prometheus_notifications_alertmanagers_discovered{%(prometheusSelector)s}[5m]) < 1
83 ||| % $._config,
84 'for': '10m',
85 labels: {
86 severity: 'warning',
87 },
88 annotations: {
89 summary: 'Prometheus is not connected to any Alertmanagers.',
90 description: 'Prometheus %(prometheusName)s is not connected to any Alertmanagers.' % $._config,
91 },
92 },
93 {
94 alert: 'PrometheusTSDBReloadsFailing',
95 expr: |||
96 increase(prometheus_tsdb_reloads_failures_total{%(prometheusSelector)s}[3h]) > 0
97 ||| % $._config,
98 'for': '4h',
99 labels: {
100 severity: 'warning',
101 },
102 annotations: {
103 summary: 'Prometheus has issues reloading blocks from disk.',
104 description: 'Prometheus %(prometheusName)s has detected {{$value | humanize}} reload failures over the last 3h.' % $._config,
105 },
106 },
107 {
108 alert: 'PrometheusTSDBCompactionsFailing',
109 expr: |||
110 increase(prometheus_tsdb_compactions_failed_total{%(prometheusSelector)s}[3h]) > 0
111 ||| % $._config,
112 'for': '4h',
113 labels: {
114 severity: 'warning',
115 },
116 annotations: {
117 summary: 'Prometheus has issues compacting blocks.',
118 description: 'Prometheus %(prometheusName)s has detected {{$value | humanize}} compaction failures over the last 3h.' % $._config,
119 },
120 },
121 {
122 alert: 'PrometheusNotIngestingSamples',
123 expr: |||
124 (
125 sum without(type) (rate(prometheus_tsdb_head_samples_appended_total{%(prometheusSelector)s}[5m])) <= 0
126 and
127 (
128 sum without(scrape_job) (prometheus_target_metadata_cache_entries{%(prometheusSelector)s}) > 0
129 or
130 sum without(rule_group) (prometheus_rule_group_rules{%(prometheusSelector)s}) > 0
131 )
132 )
133 ||| % $._config,
134 'for': '10m',
135 labels: {
136 severity: 'warning',
137 },
138 annotations: {
139 summary: 'Prometheus is not ingesting samples.',
140 description: 'Prometheus %(prometheusName)s is not ingesting samples.' % $._config,
141 },
142 },
143 {
144 alert: 'PrometheusDuplicateTimestamps',
145 expr: |||
146 rate(prometheus_target_scrapes_sample_duplicate_timestamp_total{%(prometheusSelector)s}[5m]) > 0
147 ||| % $._config,
148 'for': '10m',
149 labels: {
150 severity: 'warning',
151 },
152 annotations: {
153 summary: 'Prometheus is dropping samples with duplicate timestamps.',
154 description: 'Prometheus %(prometheusName)s is dropping {{ printf "%%.4g" $value }} samples/s with different values but duplicated timestamp.' % $._config,
155 },
156 },
157 {
158 alert: 'PrometheusOutOfOrderTimestamps',
159 expr: |||
160 rate(prometheus_target_scrapes_sample_out_of_order_total{%(prometheusSelector)s}[5m]) > 0
161 ||| % $._config,
162 'for': '10m',
163 labels: {
164 severity: 'warning',
165 },
166 annotations: {
167 summary: 'Prometheus drops samples with out-of-order timestamps.',
168 description: 'Prometheus %(prometheusName)s is dropping {{ printf "%%.4g" $value }} samples/s with timestamps arriving out of order.' % $._config,
169 },
170 },
171 {
172 alert: 'PrometheusRemoteStorageFailures',
173 expr: |||
174 (
175 (rate(prometheus_remote_storage_failed_samples_total{%(prometheusSelector)s}[5m]) or rate(prometheus_remote_storage_samples_failed_total{%(prometheusSelector)s}[5m]))
176 /
177 (
178 (rate(prometheus_remote_storage_failed_samples_total{%(prometheusSelector)s}[5m]) or rate(prometheus_remote_storage_samples_failed_total{%(prometheusSelector)s}[5m]))
179 +
180 (rate(prometheus_remote_storage_succeeded_samples_total{%(prometheusSelector)s}[5m]) or rate(prometheus_remote_storage_samples_total{%(prometheusSelector)s}[5m]))
181 )
182 )
183 * 100
184 > 1
185 ||| % $._config,
186 'for': '15m',
187 labels: {
188 severity: 'critical',
189 },
190 annotations: {
191 summary: 'Prometheus fails to send samples to remote storage.',
192 description: 'Prometheus %(prometheusName)s failed to send {{ printf "%%.1f" $value }}%% of the samples to {{ $labels.remote_name}}:{{ $labels.url }}' % $._config,
193 },
194 },
195 {
196 alert: 'PrometheusRemoteWriteBehind',
197 expr: |||
198 # Without max_over_time, failed scrapes could create false negatives, see
199 # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details.
200 (
201 max_over_time(prometheus_remote_storage_highest_timestamp_in_seconds{%(prometheusSelector)s}[5m])
202 - ignoring(remote_name, url) group_right
203 max_over_time(prometheus_remote_storage_queue_highest_sent_timestamp_seconds{%(prometheusSelector)s}[5m])
204 )
205 > 120
206 ||| % $._config,
207 'for': '15m',
208 labels: {
209 severity: 'critical',
210 },
211 annotations: {
212 summary: 'Prometheus remote write is behind.',
213 description: 'Prometheus %(prometheusName)s remote write is {{ printf "%%.1f" $value }}s behind for {{ $labels.remote_name}}:{{ $labels.url }}.' % $._config,
214 },
215 },
216 {
217 alert: 'PrometheusRemoteWriteDesiredShards',
218 expr: |||
219 # Without max_over_time, failed scrapes could create false negatives, see
220 # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details.
221 (
222 max_over_time(prometheus_remote_storage_shards_desired{%(prometheusSelector)s}[5m])
223 >
224 max_over_time(prometheus_remote_storage_shards_max{%(prometheusSelector)s}[5m])
225 )
226 ||| % $._config,
227 'for': '15m',
228 labels: {
229 severity: 'warning',
230 },
231 annotations: {
232 summary: 'Prometheus remote write desired shards calculation wants to run more than configured max shards.',
233 description: 'Prometheus %(prometheusName)s remote write desired shards calculation wants to run {{ $value }} shards for queue {{ $labels.remote_name}}:{{ $labels.url }}, which is more than the max of {{ printf `prometheus_remote_storage_shards_max{instance="%%s",%(prometheusSelector)s}` $labels.instance | query | first | value }}.' % $._config,
234 },
235 },
236 {
237 alert: 'PrometheusRuleFailures',
238 expr: |||
239 increase(prometheus_rule_evaluation_failures_total{%(prometheusSelector)s}[5m]) > 0
240 ||| % $._config,
241 'for': '15m',
242 labels: {
243 severity: 'critical',
244 },
245 annotations: {
246 summary: 'Prometheus is failing rule evaluations.',
247 description: 'Prometheus %(prometheusName)s has failed to evaluate {{ printf "%%.0f" $value }} rules in the last 5m.' % $._config,
248 },
249 },
250 {
251 alert: 'PrometheusMissingRuleEvaluations',
252 expr: |||
253 increase(prometheus_rule_group_iterations_missed_total{%(prometheusSelector)s}[5m]) > 0
254 ||| % $._config,
255 'for': '15m',
256 labels: {
257 severity: 'warning',
258 },
259 annotations: {
260 summary: 'Prometheus is missing rule evaluations due to slow rule group evaluation.',
261 description: 'Prometheus %(prometheusName)s has missed {{ printf "%%.0f" $value }} rule group evaluations in the last 5m.' % $._config,
262 },
263 },
264 {
265 alert: 'PrometheusTargetLimitHit',
266 expr: |||
267 increase(prometheus_target_scrape_pool_exceeded_target_limit_total{%(prometheusSelector)s}[5m]) > 0
268 ||| % $._config,
269 'for': '15m',
270 labels: {
271 severity: 'warning',
272 },
273 annotations: {
274 summary: 'Prometheus has dropped targets because some scrape configs have exceeded the targets limit.',
275 description: 'Prometheus %(prometheusName)s has dropped {{ printf "%%.0f" $value }} targets because the number of targets exceeded the configured target_limit.' % $._config,
276 },
277 },
278 {
279 alert: 'PrometheusLabelLimitHit',
280 expr: |||
281 increase(prometheus_target_scrape_pool_exceeded_label_limits_total{%(prometheusSelector)s}[5m]) > 0
282 ||| % $._config,
283 'for': '15m',
284 labels: {
285 severity: 'warning',
286 },
287 annotations: {
288 summary: 'Prometheus has dropped targets because some scrape configs have exceeded the labels limit.',
289 description: 'Prometheus %(prometheusName)s has dropped {{ printf "%%.0f" $value }} targets because some samples exceeded the configured label_limit, label_name_length_limit or label_value_length_limit.' % $._config,
290 },
291 },
292 {
293 alert: 'PrometheusScrapeBodySizeLimitHit',
294 expr: |||
295 increase(prometheus_target_scrapes_exceeded_body_size_limit_total{%(prometheusSelector)s}[5m]) > 0
296 ||| % $._config,
297 'for': '15m',
298 labels: {
299 severity: 'warning',
300 },
301 annotations: {
302 summary: 'Prometheus has dropped some targets that exceeded body size limit.',
303 description: 'Prometheus %(prometheusName)s has failed {{ printf "%%.0f" $value }} scrapes in the last 5m because some targets exceeded the configured body_size_limit.' % $._config,
304 },
305 },
306 {
307 alert: 'PrometheusScrapeSampleLimitHit',
308 expr: |||
309 increase(prometheus_target_scrapes_exceeded_sample_limit_total{%(prometheusSelector)s}[5m]) > 0
310 ||| % $._config,
311 'for': '15m',
312 labels: {
313 severity: 'warning',
314 },
315 annotations: {
316 summary: 'Prometheus has failed scrapes that have exceeded the configured sample limit.',
317 description: 'Prometheus %(prometheusName)s has failed {{ printf "%%.0f" $value }} scrapes in the last 5m because some targets exceeded the configured sample_limit.' % $._config,
318 },
319 },
320 {
321 alert: 'PrometheusTargetSyncFailure',
322 expr: |||
323 increase(prometheus_target_sync_failed_total{%(prometheusSelector)s}[30m]) > 0
324 ||| % $._config,
325 'for': '5m',
326 labels: {
327 severity: 'critical',
328 },
329 annotations: {
330 summary: 'Prometheus has failed to sync targets.',
331 description: '{{ printf "%%.0f" $value }} targets in Prometheus %(prometheusName)s have failed to sync because invalid configuration was supplied.' % $._config,
332 },
333 },
334 {
335 alert: 'PrometheusHighQueryLoad',
336 expr: |||
337 avg_over_time(prometheus_engine_queries{%(prometheusSelector)s}[5m]) / max_over_time(prometheus_engine_queries_concurrent_max{%(prometheusSelector)s}[5m]) > 0.8
338 ||| % $._config,
339 'for': '15m',
340 labels: {
341 severity: 'warning',
342 },
343 annotations: {
344 summary: 'Prometheus is reaching its maximum capacity serving concurrent requests.',
345 description: 'Prometheus %(prometheusName)s query API has less than 20%% available capacity in its query engine for the last 15 minutes.' % $._config,
346 },
347 },
348 ] + if $._config.prometheusHAGroupLabels == '' then self.rulesWithoutHA else self.rulesWithHA,
349 rulesWithoutHA:: [
350 {
351 alert: 'PrometheusErrorSendingAlertsToAnyAlertmanager',
352 expr: |||
353 min without (alertmanager) (
354 rate(prometheus_notifications_errors_total{%(prometheusSelector)s,alertmanager!~`%(nonNotifyingAlertmanagerRegEx)s`}[5m])
355 /
356 rate(prometheus_notifications_sent_total{%(prometheusSelector)s,alertmanager!~`%(nonNotifyingAlertmanagerRegEx)s`}[5m])
357 )
358 * 100
359 > 3
360 ||| % $._config,
361 'for': '15m',
362 labels: {
363 severity: 'critical',
364 },
365 annotations: {
366 summary: 'Prometheus encounters more than 3% errors sending alerts to any Alertmanager.',
367 description: '{{ printf "%%.1f" $value }}%% minimum errors while sending alerts from Prometheus %(prometheusName)s to any Alertmanager.' % $._config,
368 },
369 },
370 ],
371 rulesWithHA:: [
372 {
373 alert: 'PrometheusErrorSendingAlertsToAnyAlertmanager',
374 expr: |||
375 min by (%(prometheusHAGroupLabels)s) (
376 rate(prometheus_notifications_errors_total{%(prometheusSelector)s,alertmanager!~`%(nonNotifyingAlertmanagerRegEx)s`}[5m])
377 /
378 rate(prometheus_notifications_sent_total{%(prometheusSelector)s,alertmanager!~`%(nonNotifyingAlertmanagerRegEx)s`}[5m])
379 )
380 * 100
381 > 3
382 ||| % $._config,
383 'for': '15m',
384 labels: {
385 severity: 'critical',
386 },
387 annotations: {
388 summary: 'Each Prometheus server in an HA group encounters more than 3% errors sending alerts to any Alertmanager.',
389 description: '{{ printf "%%.1f" $value }}%% minimum errors while sending alerts from any Prometheus server in HA group %(prometheusHAGroupName)s to any Alertmanager.' % $._config,
390 },
391 },
392 {
393 alert: 'PrometheusHAGroupNotIngestingSamples',
394 expr: |||
395 max by (%(prometheusHAGroupLabels)s) (
396 rate(prometheus_tsdb_head_samples_appended_total{%(prometheusSelector)s}[5m])
397 and
398 (
399 sum without(scrape_job) (prometheus_target_metadata_cache_entries{%(prometheusSelector)s}) > 0
400 or
401 sum without(rule_group) (prometheus_rule_group_rules{%(prometheusSelector)s}) > 0
402 )
403 )
404 <= 0
405 ||| % $._config,
406 'for': '10m',
407 labels: {
408 severity: 'critical',
409 },
410 annotations: {
411 summary: 'A whole Prometheus HA group is not ingesting samples.',
412 description: 'None of the Prometheus instances in HA group %(prometheusHAGroupName)s is ingesting any samples.' % $._config,
413 },
414 },
415 // Both the following critical alerts, PrometheusHAGroupDown and
416 // PrometheusHAGroupCrashlooping, fire if a whole HA group is
417 // unhealthy. It is implied that a generic warning alert is in place
418 // for individual instances being down or crashlooping.
419 {
420 alert: 'PrometheusHAGroupDown',
421 expr: |||
422 (
423 count by (%(prometheusHAGroupLabels)s) (
424 avg_over_time(up{%(prometheusSelector)s}[5m]) < 0.5
425 )
426 /
427 count by (%(prometheusHAGroupLabels)s) (
428 up{%(prometheusSelector)s}
429 )
430 )
431 > 0.5
432 ||| % $._config,
433 'for': '5m',
434 labels: {
435 severity: 'critical',
436 },
437 annotations: {
438 summary: 'More than half of the Prometheus instances within the same HA group are down.',
439 description: '{{ $value | humanizePercentage }} of Prometheus instances within the %(prometheusHAGroupName)s HA group have been up for less than half of the last 5m.' % $._config,
440 },
441 },
442 {
443 alert: 'PrometheusHAGroupCrashlooping',
444 expr: |||
445 (
446 prometheus_tsdb_clean_start{%(prometheusSelector)s} == 0
447 and
448 (
449 count by (%(prometheusHAGroupLabels)s) (
450 changes(process_start_time_seconds{%(prometheusSelector)s}[1h]) > 1
451 )
452 /
453 count by (%(prometheusHAGroupLabels)s) (
454 up{%(prometheusSelector)s}
455 )
456 )
457 > 0.5
458 )
459 or
460 (
461 count by (%(prometheusHAGroupLabels)s) (
462 changes(process_start_time_seconds{%(prometheusSelector)s}[30m]) > 4
463 )
464 /
465 count by (%(prometheusHAGroupLabels)s) (
466 up{%(prometheusSelector)s}
467 )
468 )
469 > 0.5
470 ||| % $._config,
471 'for': '5m',
472 labels: {
473 severity: 'critical',
474 },
475 annotations: {
476 summary: 'More than half of the Prometheus instances within the same HA group are crashlooping.',
477 description: '{{ $value | humanizePercentage }} of Prometheus instances within the %(prometheusHAGroupName)s HA group have had at least 5 total restarts in the last 30m or 2 unclean restarts in the last 1h.' % $._config,
478 },
479 },
480 ],
481 },
482 ],
483 },
484 }