Debian Codegraph @

contrib/mixin/mixin.libsonnet

raw
1 {
2 _config+:: {
3 etcd_selector: 'job=~".*etcd.*"',
4 // etcd_instance_labels are the label names that are uniquely
5 // identifying an instance and need to be aggreated away for alerts
6 // that are about an etcd cluster as a whole. For example, if etcd
7 // instances are deployed on K8s, you will likely want to change
8 // this to 'instance, pod'.
9 etcd_instance_labels: 'instance',
10 // scrape_interval_seconds is the global scrape interval which can be
11 // used to dynamically adjust rate windows as a function of the interval.
12 scrape_interval_seconds: 30,
13 // Dashboard variable refresh option on Grafana (https://grafana.com/docs/grafana/latest/datasources/prometheus/).
14 // 0 : Never (Will never refresh the Dashboard variables values)
15 // 1 : On Dashboard Load (Will refresh Dashboards variables when dashboard are loaded)
16 // 2 : On Time Range Change (Will refresh Dashboards variables when time range will be changed)
17 dashboard_var_refresh: 2,
18 },
19
20 prometheusAlerts+:: {
21 groups+: [
22 {
23 name: 'etcd',
24 rules: [
25 {
26 alert: 'etcdMembersDown',
27 expr: |||
28 max without (endpoint) (
29 sum without (%(etcd_instance_labels)s) (up{%(etcd_selector)s} == bool 0)
30 or
31 count without (To) (
32 sum without (%(etcd_instance_labels)s) (rate(etcd_network_peer_sent_failures_total{%(etcd_selector)s}[%(network_failure_range)ss])) > 0.01
33 )
34 )
35 > 0
36 ||| % {etcd_instance_labels: $._config.etcd_instance_labels, etcd_selector: $._config.etcd_selector, network_failure_range: $._config.scrape_interval_seconds*4},
37 'for': '10m',
38 labels: {
39 severity: 'critical',
40 },
41 annotations: {
42 description: 'etcd cluster "{{ $labels.job }}": members are down ({{ $value }}).',
43 summary: 'etcd cluster members are down.',
44 },
45 },
46 {
47 alert: 'etcdInsufficientMembers',
48 expr: |||
49 sum(up{%(etcd_selector)s} == bool 1) without (%(etcd_instance_labels)s) < ((count(up{%(etcd_selector)s}) without (%(etcd_instance_labels)s) + 1) / 2)
50 ||| % $._config,
51 'for': '3m',
52 labels: {
53 severity: 'critical',
54 },
55 annotations: {
56 description: 'etcd cluster "{{ $labels.job }}": insufficient members ({{ $value }}).',
57 summary: 'etcd cluster has insufficient number of members.',
58 },
59 },
60 {
61 alert: 'etcdNoLeader',
62 expr: |||
63 etcd_server_has_leader{%(etcd_selector)s} == 0
64 ||| % $._config,
65 'for': '1m',
66 labels: {
67 severity: 'critical',
68 },
69 annotations: {
70 description: 'etcd cluster "{{ $labels.job }}": member {{ $labels.instance }} has no leader.',
71 summary: 'etcd cluster has no leader.',
72 },
73 },
74 {
75 alert: 'etcdHighNumberOfLeaderChanges',
76 expr: |||
77 increase((max without (%(etcd_instance_labels)s) (etcd_server_leader_changes_seen_total{%(etcd_selector)s}) or 0*absent(etcd_server_leader_changes_seen_total{%(etcd_selector)s}))[15m:1m]) >= 4
78 ||| % $._config,
79 'for': '5m',
80 labels: {
81 severity: 'warning',
82 },
83 annotations: {
84 description: 'etcd cluster "{{ $labels.job }}": {{ $value }} leader changes within the last 15 minutes. Frequent elections may be a sign of insufficient resources, high network latency, or disruptions by other components and should be investigated.',
85 summary: 'etcd cluster has high number of leader changes.',
86 },
87 },
88 {
89 alert: 'etcdHighNumberOfFailedGRPCRequests',
90 expr: |||
91 100 * sum(rate(grpc_server_handled_total{%(etcd_selector)s, grpc_code!="OK"}[5m])) without (grpc_type, grpc_code)
92 /
93 sum(rate(grpc_server_handled_total{%(etcd_selector)s}[5m])) without (grpc_type, grpc_code)
94 > 1
95 ||| % $._config,
96 'for': '10m',
97 labels: {
98 severity: 'warning',
99 },
100 annotations: {
101 description: 'etcd cluster "{{ $labels.job }}": {{ $value }}% of requests for {{ $labels.grpc_method }} failed on etcd instance {{ $labels.instance }}.',
102 summary: 'etcd cluster has high number of failed grpc requests.',
103 },
104 },
105 {
106 alert: 'etcdHighNumberOfFailedGRPCRequests',
107 expr: |||
108 100 * sum(rate(grpc_server_handled_total{%(etcd_selector)s, grpc_code!="OK"}[5m])) without (grpc_type, grpc_code)
109 /
110 sum(rate(grpc_server_handled_total{%(etcd_selector)s}[5m])) without (grpc_type, grpc_code)
111 > 5
112 ||| % $._config,
113 'for': '5m',
114 labels: {
115 severity: 'critical',
116 },
117 annotations: {
118 description: 'etcd cluster "{{ $labels.job }}": {{ $value }}% of requests for {{ $labels.grpc_method }} failed on etcd instance {{ $labels.instance }}.',
119 summary: 'etcd cluster has high number of failed grpc requests.',
120 },
121 },
122 {
123 alert: 'etcdGRPCRequestsSlow',
124 expr: |||
125 histogram_quantile(0.99, sum(rate(grpc_server_handling_seconds_bucket{%(etcd_selector)s, grpc_type="unary"}[5m])) without(grpc_type))
126 > 0.15
127 ||| % $._config,
128 'for': '10m',
129 labels: {
130 severity: 'critical',
131 },
132 annotations: {
133 description: 'etcd cluster "{{ $labels.job }}": gRPC requests to {{ $labels.grpc_method }} are taking {{ $value }}s on etcd instance {{ $labels.instance }}.',
134 summary: 'etcd grpc requests are slow',
135 },
136 },
137 {
138 alert: 'etcdMemberCommunicationSlow',
139 expr: |||
140 histogram_quantile(0.99, rate(etcd_network_peer_round_trip_time_seconds_bucket{%(etcd_selector)s}[5m]))
141 > 0.15
142 ||| % $._config,
143 'for': '10m',
144 labels: {
145 severity: 'warning',
146 },
147 annotations: {
148 description: 'etcd cluster "{{ $labels.job }}": member communication with {{ $labels.To }} is taking {{ $value }}s on etcd instance {{ $labels.instance }}.',
149 summary: 'etcd cluster member communication is slow.',
150 },
151 },
152 {
153 alert: 'etcdHighNumberOfFailedProposals',
154 expr: |||
155 rate(etcd_server_proposals_failed_total{%(etcd_selector)s}[15m]) > 5
156 ||| % $._config,
157 'for': '15m',
158 labels: {
159 severity: 'warning',
160 },
161 annotations: {
162 description: 'etcd cluster "{{ $labels.job }}": {{ $value }} proposal failures within the last 30 minutes on etcd instance {{ $labels.instance }}.',
163 summary: 'etcd cluster has high number of proposal failures.',
164 },
165 },
166 {
167 alert: 'etcdHighFsyncDurations',
168 expr: |||
169 histogram_quantile(0.99, rate(etcd_disk_wal_fsync_duration_seconds_bucket{%(etcd_selector)s}[5m]))
170 > 0.5
171 ||| % $._config,
172 'for': '10m',
173 labels: {
174 severity: 'warning',
175 },
176 annotations: {
177 description: 'etcd cluster "{{ $labels.job }}": 99th percentile fsync durations are {{ $value }}s on etcd instance {{ $labels.instance }}.',
178 summary: 'etcd cluster 99th percentile fsync durations are too high.',
179 },
180 },
181 {
182 alert: 'etcdHighFsyncDurations',
183 expr: |||
184 histogram_quantile(0.99, rate(etcd_disk_wal_fsync_duration_seconds_bucket{%(etcd_selector)s}[5m]))
185 > 1
186 ||| % $._config,
187 'for': '10m',
188 labels: {
189 severity: 'critical',
190 },
191 annotations: {
192 message: 'etcd cluster "{{ $labels.job }}": 99th percentile fsync durations are {{ $value }}s on etcd instance {{ $labels.instance }}.',
193 },
194 },
195 {
196 alert: 'etcdHighCommitDurations',
197 expr: |||
198 histogram_quantile(0.99, rate(etcd_disk_backend_commit_duration_seconds_bucket{%(etcd_selector)s}[5m]))
199 > 0.25
200 ||| % $._config,
201 'for': '10m',
202 labels: {
203 severity: 'warning',
204 },
205 annotations: {
206 description: 'etcd cluster "{{ $labels.job }}": 99th percentile commit durations {{ $value }}s on etcd instance {{ $labels.instance }}.',
207 summary: 'etcd cluster 99th percentile commit durations are too high.',
208 },
209 },
210 {
211 alert: 'etcdBackendQuotaLowSpace',
212 expr: |||
213 (etcd_mvcc_db_total_size_in_bytes/etcd_server_quota_backend_bytes)*100 > 95
214 ||| % $._config,
215 'for': '10m',
216 labels: {
217 severity: 'critical',
218 },
219 annotations: {
220 message: 'etcd cluster "{{ $labels.job }}": database size exceeds the defined quota on etcd instance {{ $labels.instance }}, please defrag or increase the quota as the writes to etcd will be disabled when it is full.',
221 },
222 },
223 {
224 alert: 'etcdExcessiveDatabaseGrowth',
225 expr: |||
226 increase(((etcd_mvcc_db_total_size_in_bytes/etcd_server_quota_backend_bytes)*100)[240m:1m]) > 50
227 ||| % $._config,
228 'for': '10m',
229 labels: {
230 severity: 'warning',
231 },
232 annotations: {
233 message: 'etcd cluster "{{ $labels.job }}": Observed surge in etcd writes leading to 50% increase in database size over the past four hours on etcd instance {{ $labels.instance }}, please check as it might be disruptive.',
234 },
235 },
236 ],
237 },
238 ],
239 },
240
241 grafanaDashboards+:: {
242 'etcd.json': {
243 uid: std.md5('etcd.json'),
244 title: 'etcd',
245 description: 'etcd sample Grafana dashboard with Prometheus',
246 tags: [ 'etcd-mixin' ],
247 style: 'dark',
248 timezone: 'browser',
249 editable: true,
250 hideControls: false,
251 sharedCrosshair: false,
252 rows: [
253 {
254 collapse: false,
255 editable: true,
256 height: '250px',
257 panels: [
258 {
259 cacheTimeout: null,
260 colorBackground: false,
261 colorValue: false,
262 colors: [
263 'rgba(245, 54, 54, 0.9)',
264 'rgba(237, 129, 40, 0.89)',
265 'rgba(50, 172, 45, 0.97)',
266 ],
267 datasource: '$datasource',
268 editable: true,
269 'error': false,
270 format: 'none',
271 gauge: {
272 maxValue: 100,
273 minValue: 0,
274 show: false,
275 thresholdLabels: false,
276 thresholdMarkers: true,
277 },
278 id: 28,
279 interval: null,
280 isNew: true,
281 links: [],
282 mappingType: 1,
283 mappingTypes: [
284 {
285 name: 'value to text',
286 value: 1,
287 },
288 {
289 name: 'range to text',
290 value: 2,
291 },
292 ],
293 maxDataPoints: 100,
294 nullPointMode: 'connected',
295 nullText: null,
296 postfix: '',
297 postfixFontSize: '50%',
298 prefix: '',
299 prefixFontSize: '50%',
300 rangeMaps: [{
301 from: 'null',
302 text: 'N/A',
303 to: 'null',
304 }],
305 span: 3,
306 sparkline: {
307 fillColor: 'rgba(31, 118, 189, 0.18)',
308 full: false,
309 lineColor: 'rgb(31, 120, 193)',
310 show: false,
311 },
312 targets: [{
313 expr: 'sum(etcd_server_has_leader{job="$cluster"})',
314 intervalFactor: 2,
315 legendFormat: '',
316 metric: 'etcd_server_has_leader',
317 refId: 'A',
318 step: 20,
319 }],
320 thresholds: '',
321 title: 'Up',
322 type: 'singlestat',
323 valueFontSize: '200%',
324 valueMaps: [{
325 op: '=',
326 text: 'N/A',
327 value: 'null',
328 }],
329 valueName: 'avg',
330 },
331 {
332 aliasColors: {},
333 bars: false,
334 datasource: '$datasource',
335 editable: true,
336 'error': false,
337 fill: 0,
338 id: 23,
339 isNew: true,
340 legend: {
341 avg: false,
342 current: false,
343 max: false,
344 min: false,
345 show: false,
346 total: false,
347 values: false,
348 },
349 lines: true,
350 linewidth: 2,
351 links: [],
352 nullPointMode: 'connected',
353 percentage: false,
354 pointradius: 5,
355 points: false,
356 renderer: 'flot',
357 seriesOverrides: [],
358 span: 5,
359 stack: false,
360 steppedLine: false,
361 targets: [
362 {
363 expr: 'sum(rate(grpc_server_started_total{job="$cluster",grpc_type="unary"}[5m]))',
364 format: 'time_series',
365 intervalFactor: 2,
366 legendFormat: 'RPC Rate',
367 metric: 'grpc_server_started_total',
368 refId: 'A',
369 step: 2,
370 },
371 {
372 expr: 'sum(rate(grpc_server_handled_total{job="$cluster",grpc_type="unary",grpc_code!="OK"}[5m]))',
373 format: 'time_series',
374 intervalFactor: 2,
375 legendFormat: 'RPC Failed Rate',
376 metric: 'grpc_server_handled_total',
377 refId: 'B',
378 step: 2,
379 },
380 ],
381 thresholds: [],
382 timeFrom: null,
383 timeShift: null,
384 title: 'RPC Rate',
385 tooltip: {
386 msResolution: false,
387 shared: true,
388 sort: 0,
389 value_type: 'individual',
390 },
391 type: 'graph',
392 xaxis: {
393 mode: 'time',
394 name: null,
395 show: true,
396 values: [],
397 },
398 yaxes: [
399 {
400 format: 'ops',
401 label: null,
402 logBase: 1,
403 max: null,
404 min: null,
405 show: true,
406 },
407 {
408 format: 'short',
409 label: null,
410 logBase: 1,
411 max: null,
412 min: null,
413 show: true,
414 },
415 ],
416 },
417 {
418 aliasColors: {},
419 bars: false,
420 datasource: '$datasource',
421 editable: true,
422 'error': false,
423 fill: 0,
424 id: 41,
425 isNew: true,
426 legend: {
427 avg: false,
428 current: false,
429 max: false,
430 min: false,
431 show: false,
432 total: false,
433 values: false,
434 },
435 lines: true,
436 linewidth: 2,
437 links: [],
438 nullPointMode: 'connected',
439 percentage: false,
440 pointradius: 5,
441 points: false,
442 renderer: 'flot',
443 seriesOverrides: [],
444 span: 4,
445 stack: true,
446 steppedLine: false,
447 targets: [
448 {
449 expr: 'sum(grpc_server_started_total{job="$cluster",grpc_service="etcdserverpb.Watch",grpc_type="bidi_stream"}) - sum(grpc_server_handled_total{job="$cluster",grpc_service="etcdserverpb.Watch",grpc_type="bidi_stream"})',
450 intervalFactor: 2,
451 legendFormat: 'Watch Streams',
452 metric: 'grpc_server_handled_total',
453 refId: 'A',
454 step: 4,
455 },
456 {
457 expr: 'sum(grpc_server_started_total{job="$cluster",grpc_service="etcdserverpb.Lease",grpc_type="bidi_stream"}) - sum(grpc_server_handled_total{job="$cluster",grpc_service="etcdserverpb.Lease",grpc_type="bidi_stream"})',
458 intervalFactor: 2,
459 legendFormat: 'Lease Streams',
460 metric: 'grpc_server_handled_total',
461 refId: 'B',
462 step: 4,
463 },
464 ],
465 thresholds: [],
466 timeFrom: null,
467 timeShift: null,
468 title: 'Active Streams',
469 tooltip: {
470 msResolution: false,
471 shared: true,
472 sort: 0,
473 value_type: 'individual',
474 },
475 type: 'graph',
476 xaxis: {
477 mode: 'time',
478 name: null,
479 show: true,
480 values: [],
481 },
482 yaxes: [
483 {
484 format: 'short',
485 label: '',
486 logBase: 1,
487 max: null,
488 min: null,
489 show: true,
490 },
491 {
492 format: 'short',
493 label: null,
494 logBase: 1,
495 max: null,
496 min: null,
497 show: true,
498 },
499 ],
500 },
501 ],
502 showTitle: false,
503 title: 'Row',
504 },
505 {
506 collapse: false,
507 editable: true,
508 height: '250px',
509 panels: [
510 {
511 aliasColors: {},
512 bars: false,
513 datasource: '$datasource',
514 decimals: null,
515 editable: true,
516 'error': false,
517 fill: 0,
518 grid: {},
519 id: 1,
520 legend: {
521 avg: false,
522 current: false,
523 max: false,
524 min: false,
525 show: false,
526 total: false,
527 values: false,
528 },
529 lines: true,
530 linewidth: 2,
531 links: [],
532 nullPointMode: 'connected',
533 percentage: false,
534 pointradius: 5,
535 points: false,
536 renderer: 'flot',
537 seriesOverrides: [],
538 span: 4,
539 stack: false,
540 steppedLine: false,
541 targets: [{
542 expr: 'etcd_mvcc_db_total_size_in_bytes{job="$cluster"}',
543 hide: false,
544 interval: '',
545 intervalFactor: 2,
546 legendFormat: '{{instance}} DB Size',
547 metric: '',
548 refId: 'A',
549 step: 4,
550 }],
551 thresholds: [],
552 timeFrom: null,
553 timeShift: null,
554 title: 'DB Size',
555 tooltip: {
556 msResolution: false,
557 shared: true,
558 sort: 0,
559 value_type: 'cumulative',
560 },
561 type: 'graph',
562 xaxis: {
563 mode: 'time',
564 name: null,
565 show: true,
566 values: [],
567 },
568 yaxes: [
569 {
570 format: 'bytes',
571 logBase: 1,
572 max: null,
573 min: null,
574 show: true,
575 },
576 {
577 format: 'short',
578 logBase: 1,
579 max: null,
580 min: null,
581 show: false,
582 },
583 ],
584 },
585 {
586 aliasColors: {},
587 bars: false,
588 datasource: '$datasource',
589 editable: true,
590 'error': false,
591 fill: 0,
592 grid: {},
593 id: 3,
594 legend: {
595 avg: false,
596 current: false,
597 max: false,
598 min: false,
599 show: false,
600 total: false,
601 values: false,
602 },
603 lines: true,
604 linewidth: 2,
605 links: [],
606 nullPointMode: 'connected',
607 percentage: false,
608 pointradius: 1,
609 points: false,
610 renderer: 'flot',
611 seriesOverrides: [],
612 span: 4,
613 stack: false,
614 steppedLine: true,
615 targets: [
616 {
617 expr: 'histogram_quantile(0.99, sum(rate(etcd_disk_wal_fsync_duration_seconds_bucket{job="$cluster"}[5m])) by (instance, le))',
618 hide: false,
619 intervalFactor: 2,
620 legendFormat: '{{instance}} WAL fsync',
621 metric: 'etcd_disk_wal_fsync_duration_seconds_bucket',
622 refId: 'A',
623 step: 4,
624 },
625 {
626 expr: 'histogram_quantile(0.99, sum(rate(etcd_disk_backend_commit_duration_seconds_bucket{job="$cluster"}[5m])) by (instance, le))',
627 intervalFactor: 2,
628 legendFormat: '{{instance}} DB fsync',
629 metric: 'etcd_disk_backend_commit_duration_seconds_bucket',
630 refId: 'B',
631 step: 4,
632 },
633 ],
634 thresholds: [],
635 timeFrom: null,
636 timeShift: null,
637 title: 'Disk Sync Duration',
638 tooltip: {
639 msResolution: false,
640 shared: true,
641 sort: 0,
642 value_type: 'cumulative',
643 },
644 type: 'graph',
645 xaxis: {
646 mode: 'time',
647 name: null,
648 show: true,
649 values: [],
650 },
651 yaxes: [
652 {
653 format: 's',
654 logBase: 1,
655 max: null,
656 min: null,
657 show: true,
658 },
659 {
660 format: 'short',
661 logBase: 1,
662 max: null,
663 min: null,
664 show: false,
665 },
666 ],
667 },
668 {
669 aliasColors: {},
670 bars: false,
671 datasource: '$datasource',
672 editable: true,
673 'error': false,
674 fill: 0,
675 id: 29,
676 isNew: true,
677 legend: {
678 avg: false,
679 current: false,
680 max: false,
681 min: false,
682 show: false,
683 total: false,
684 values: false,
685 },
686 lines: true,
687 linewidth: 2,
688 links: [],
689 nullPointMode: 'connected',
690 percentage: false,
691 pointradius: 5,
692 points: false,
693 renderer: 'flot',
694 seriesOverrides: [],
695 span: 4,
696 stack: false,
697 steppedLine: false,
698 targets: [{
699 expr: 'process_resident_memory_bytes{job="$cluster"}',
700 intervalFactor: 2,
701 legendFormat: '{{instance}} Resident Memory',
702 metric: 'process_resident_memory_bytes',
703 refId: 'A',
704 step: 4,
705 }],
706 thresholds: [],
707 timeFrom: null,
708 timeShift: null,
709 title: 'Memory',
710 tooltip: {
711 msResolution: false,
712 shared: true,
713 sort: 0,
714 value_type: 'individual',
715 },
716 type: 'graph',
717 xaxis: {
718 mode: 'time',
719 name: null,
720 show: true,
721 values: [],
722 },
723 yaxes: [
724 {
725 format: 'bytes',
726 label: null,
727 logBase: 1,
728 max: null,
729 min: null,
730 show: true,
731 },
732 {
733 format: 'short',
734 label: null,
735 logBase: 1,
736 max: null,
737 min: null,
738 show: true,
739 },
740 ],
741 },
742 ],
743 title: 'New row',
744 },
745 {
746 collapse: false,
747 editable: true,
748 height: '250px',
749 panels: [
750 {
751 aliasColors: {},
752 bars: false,
753 datasource: '$datasource',
754 editable: true,
755 'error': false,
756 fill: 5,
757 id: 22,
758 isNew: true,
759 legend: {
760 avg: false,
761 current: false,
762 max: false,
763 min: false,
764 show: false,
765 total: false,
766 values: false,
767 },
768 lines: true,
769 linewidth: 2,
770 links: [],
771 nullPointMode: 'connected',
772 percentage: false,
773 pointradius: 5,
774 points: false,
775 renderer: 'flot',
776 seriesOverrides: [],
777 span: 3,
778 stack: true,
779 steppedLine: false,
780 targets: [{
781 expr: 'rate(etcd_network_client_grpc_received_bytes_total{job="$cluster"}[5m])',
782 intervalFactor: 2,
783 legendFormat: '{{instance}} Client Traffic In',
784 metric: 'etcd_network_client_grpc_received_bytes_total',
785 refId: 'A',
786 step: 4,
787 }],
788 thresholds: [],
789 timeFrom: null,
790 timeShift: null,
791 title: 'Client Traffic In',
792 tooltip: {
793 msResolution: false,
794 shared: true,
795 sort: 0,
796 value_type: 'individual',
797 },
798 type: 'graph',
799 xaxis: {
800 mode: 'time',
801 name: null,
802 show: true,
803 values: [],
804 },
805 yaxes: [
806 {
807 format: 'Bps',
808 label: null,
809 logBase: 1,
810 max: null,
811 min: null,
812 show: true,
813 },
814 {
815 format: 'short',
816 label: null,
817 logBase: 1,
818 max: null,
819 min: null,
820 show: true,
821 },
822 ],
823 },
824 {
825 aliasColors: {},
826 bars: false,
827 datasource: '$datasource',
828 editable: true,
829 'error': false,
830 fill: 5,
831 id: 21,
832 isNew: true,
833 legend: {
834 avg: false,
835 current: false,
836 max: false,
837 min: false,
838 show: false,
839 total: false,
840 values: false,
841 },
842 lines: true,
843 linewidth: 2,
844 links: [],
845 nullPointMode: 'connected',
846 percentage: false,
847 pointradius: 5,
848 points: false,
849 renderer: 'flot',
850 seriesOverrides: [],
851 span: 3,
852 stack: true,
853 steppedLine: false,
854 targets: [{
855 expr: 'rate(etcd_network_client_grpc_sent_bytes_total{job="$cluster"}[5m])',
856 intervalFactor: 2,
857 legendFormat: '{{instance}} Client Traffic Out',
858 metric: 'etcd_network_client_grpc_sent_bytes_total',
859 refId: 'A',
860 step: 4,
861 }],
862 thresholds: [],
863 timeFrom: null,
864 timeShift: null,
865 title: 'Client Traffic Out',
866 tooltip: {
867 msResolution: false,
868 shared: true,
869 sort: 0,
870 value_type: 'individual',
871 },
872 type: 'graph',
873 xaxis: {
874 mode: 'time',
875 name: null,
876 show: true,
877 values: [],
878 },
879 yaxes: [
880 {
881 format: 'Bps',
882 label: null,
883 logBase: 1,
884 max: null,
885 min: null,
886 show: true,
887 },
888 {
889 format: 'short',
890 label: null,
891 logBase: 1,
892 max: null,
893 min: null,
894 show: true,
895 },
896 ],
897 },
898 {
899 aliasColors: {},
900 bars: false,
901 datasource: '$datasource',
902 editable: true,
903 'error': false,
904 fill: 0,
905 id: 20,
906 isNew: true,
907 legend: {
908 avg: false,
909 current: false,
910 max: false,
911 min: false,
912 show: false,
913 total: false,
914 values: false,
915 },
916 lines: true,
917 linewidth: 2,
918 links: [],
919 nullPointMode: 'connected',
920 percentage: false,
921 pointradius: 5,
922 points: false,
923 renderer: 'flot',
924 seriesOverrides: [],
925 span: 3,
926 stack: false,
927 steppedLine: false,
928 targets: [{
929 expr: 'sum(rate(etcd_network_peer_received_bytes_total{job="$cluster"}[5m])) by (instance)',
930 intervalFactor: 2,
931 legendFormat: '{{instance}} Peer Traffic In',
932 metric: 'etcd_network_peer_received_bytes_total',
933 refId: 'A',
934 step: 4,
935 }],
936 thresholds: [],
937 timeFrom: null,
938 timeShift: null,
939 title: 'Peer Traffic In',
940 tooltip: {
941 msResolution: false,
942 shared: true,
943 sort: 0,
944 value_type: 'individual',
945 },
946 type: 'graph',
947 xaxis: {
948 mode: 'time',
949 name: null,
950 show: true,
951 values: [],
952 },
953 yaxes: [
954 {
955 format: 'Bps',
956 label: null,
957 logBase: 1,
958 max: null,
959 min: null,
960 show: true,
961 },
962 {
963 format: 'short',
964 label: null,
965 logBase: 1,
966 max: null,
967 min: null,
968 show: true,
969 },
970 ],
971 },
972 {
973 aliasColors: {},
974 bars: false,
975 datasource: '$datasource',
976 decimals: null,
977 editable: true,
978 'error': false,
979 fill: 0,
980 grid: {},
981 id: 16,
982 legend: {
983 avg: false,
984 current: false,
985 max: false,
986 min: false,
987 show: false,
988 total: false,
989 values: false,
990 },
991 lines: true,
992 linewidth: 2,
993 links: [],
994 nullPointMode: 'connected',
995 percentage: false,
996 pointradius: 5,
997 points: false,
998 renderer: 'flot',
999 seriesOverrides: [],
1000 span: 3,
1001 stack: false,
1002 steppedLine: false,
1003 targets: [{
1004 expr: 'sum(rate(etcd_network_peer_sent_bytes_total{job="$cluster"}[5m])) by (instance)',
1005 hide: false,
1006 interval: '',
1007 intervalFactor: 2,
1008 legendFormat: '{{instance}} Peer Traffic Out',
1009 metric: 'etcd_network_peer_sent_bytes_total',
1010 refId: 'A',
1011 step: 4,
1012 }],
1013 thresholds: [],
1014 timeFrom: null,
1015 timeShift: null,
1016 title: 'Peer Traffic Out',
1017 tooltip: {
1018 msResolution: false,
1019 shared: true,
1020 sort: 0,
1021 value_type: 'cumulative',
1022 },
1023 type: 'graph',
1024 xaxis: {
1025 mode: 'time',
1026 name: null,
1027 show: true,
1028 values: [],
1029 },
1030 yaxes: [
1031 {
1032 format: 'Bps',
1033 logBase: 1,
1034 max: null,
1035 min: null,
1036 show: true,
1037 },
1038 {
1039 format: 'short',
1040 logBase: 1,
1041 max: null,
1042 min: null,
1043 show: true,
1044 },
1045 ],
1046 },
1047 ],
1048 title: 'New row',
1049 },
1050 {
1051 collapse: false,
1052 editable: true,
1053 height: '250px',
1054 panels: [
1055 {
1056 aliasColors: {},
1057 bars: false,
1058 datasource: '$datasource',
1059 editable: true,
1060 'error': false,
1061 fill: 0,
1062 id: 40,
1063 isNew: true,
1064 legend: {
1065 avg: false,
1066 current: false,
1067 max: false,
1068 min: false,
1069 show: false,
1070 total: false,
1071 values: false,
1072 },
1073 lines: true,
1074 linewidth: 2,
1075 links: [],
1076 nullPointMode: 'connected',
1077 percentage: false,
1078 pointradius: 5,
1079 points: false,
1080 renderer: 'flot',
1081 seriesOverrides: [],
1082 span: 6,
1083 stack: false,
1084 steppedLine: false,
1085 targets: [
1086 {
1087 expr: 'sum(rate(etcd_server_proposals_failed_total{job="$cluster"}[5m]))',
1088 intervalFactor: 2,
1089 legendFormat: 'Proposal Failure Rate',
1090 metric: 'etcd_server_proposals_failed_total',
1091 refId: 'A',
1092 step: 2,
1093 },
1094 {
1095 expr: 'sum(etcd_server_proposals_pending{job="$cluster"})',
1096 intervalFactor: 2,
1097 legendFormat: 'Proposal Pending Total',
1098 metric: 'etcd_server_proposals_pending',
1099 refId: 'B',
1100 step: 2,
1101 },
1102 {
1103 expr: 'sum(rate(etcd_server_proposals_committed_total{job="$cluster"}[5m]))',
1104 intervalFactor: 2,
1105 legendFormat: 'Proposal Commit Rate',
1106 metric: 'etcd_server_proposals_committed_total',
1107 refId: 'C',
1108 step: 2,
1109 },
1110 {
1111 expr: 'sum(rate(etcd_server_proposals_applied_total{job="$cluster"}[5m]))',
1112 intervalFactor: 2,
1113 legendFormat: 'Proposal Apply Rate',
1114 refId: 'D',
1115 step: 2,
1116 },
1117 ],
1118 thresholds: [],
1119 timeFrom: null,
1120 timeShift: null,
1121 title: 'Raft Proposals',
1122 tooltip: {
1123 msResolution: false,
1124 shared: true,
1125 sort: 0,
1126 value_type: 'individual',
1127 },
1128 type: 'graph',
1129 xaxis: {
1130 mode: 'time',
1131 name: null,
1132 show: true,
1133 values: [],
1134 },
1135 yaxes: [
1136 {
1137 format: 'short',
1138 label: '',
1139 logBase: 1,
1140 max: null,
1141 min: null,
1142 show: true,
1143 },
1144 {
1145 format: 'short',
1146 label: null,
1147 logBase: 1,
1148 max: null,
1149 min: null,
1150 show: true,
1151 },
1152 ],
1153 },
1154 {
1155 aliasColors: {},
1156 bars: false,
1157 datasource: '$datasource',
1158 decimals: 0,
1159 editable: true,
1160 'error': false,
1161 fill: 0,
1162 id: 19,
1163 isNew: true,
1164 legend: {
1165 alignAsTable: false,
1166 avg: false,
1167 current: false,
1168 max: false,
1169 min: false,
1170 rightSide: false,
1171 show: false,
1172 total: false,
1173 values: false,
1174 },
1175 lines: true,
1176 linewidth: 2,
1177 links: [],
1178 nullPointMode: 'connected',
1179 percentage: false,
1180 pointradius: 5,
1181 points: false,
1182 renderer: 'flot',
1183 seriesOverrides: [],
1184 span: 6,
1185 stack: false,
1186 steppedLine: false,
1187 targets: [{
1188 expr: 'changes(etcd_server_leader_changes_seen_total{job="$cluster"}[1d])',
1189 intervalFactor: 2,
1190 legendFormat: '{{instance}} Total Leader Elections Per Day',
1191 metric: 'etcd_server_leader_changes_seen_total',
1192 refId: 'A',
1193 step: 2,
1194 }],
1195 thresholds: [],
1196 timeFrom: null,
1197 timeShift: null,
1198 title: 'Total Leader Elections Per Day',
1199 tooltip: {
1200 msResolution: false,
1201 shared: true,
1202 sort: 0,
1203 value_type: 'individual',
1204 },
1205 type: 'graph',
1206 xaxis: {
1207 mode: 'time',
1208 name: null,
1209 show: true,
1210 values: [],
1211 },
1212 yaxes: [
1213 {
1214 format: 'short',
1215 label: null,
1216 logBase: 1,
1217 max: null,
1218 min: null,
1219 show: true,
1220 },
1221 {
1222 format: 'short',
1223 label: null,
1224 logBase: 1,
1225 max: null,
1226 min: null,
1227 show: true,
1228 },
1229 ],
1230 },
1231 ],
1232 title: 'New row',
1233 },
1234 ],
1235 time: {
1236 from: 'now-15m',
1237 to: 'now',
1238 },
1239 timepicker: {
1240 now: true,
1241 refresh_intervals: [
1242 '5s',
1243 '10s',
1244 '30s',
1245 '1m',
1246 '5m',
1247 '15m',
1248 '30m',
1249 '1h',
1250 '2h',
1251 '1d',
1252 ],
1253 time_options: [
1254 '5m',
1255 '15m',
1256 '1h',
1257 '6h',
1258 '12h',
1259 '24h',
1260 '2d',
1261 '7d',
1262 '30d',
1263 ],
1264 },
1265 templating: {
1266 list: [
1267 {
1268 current: {
1269 text: 'Prometheus',
1270 value: 'Prometheus',
1271 },
1272 hide: 0,
1273 label: null,
1274 name: 'datasource',
1275 options: [],
1276 query: 'prometheus',
1277 refresh: 1,
1278 regex: '',
1279 type: 'datasource',
1280 },
1281 {
1282 allValue: null,
1283 current: {
1284 text: 'prod',
1285 value: 'prod',
1286 },
1287 datasource: '$datasource',
1288 hide: 0,
1289 includeAll: false,
1290 label: 'cluster',
1291 multi: false,
1292 name: 'cluster',
1293 options: [],
1294 query: 'label_values(etcd_server_has_leader, job)',
1295 refresh: $._config.dashboard_var_refresh,
1296 regex: '',
1297 sort: 2,
1298 tagValuesQuery: '',
1299 tags: [],
1300 tagsQuery: '',
1301 type: 'query',
1302 useTags: false,
1303 },
1304 ],
1305 },
1306 annotations: {
1307 list: [],
1308 },
1309 refresh: '10s',
1310 schemaVersion: 13,
1311 version: 215,
1312 links: [],
1313 gnetId: null,
1314 },
1315 },
1316 }