Debian Codegraph @

documentation/prometheus-mixin/dashboards.libsonnet

raw
1 local grafana = import 'github.com/grafana/grafonnet-lib/grafonnet/grafana.libsonnet';
2 local g = import 'github.com/grafana/jsonnet-libs/grafana-builder/grafana.libsonnet';
3 local dashboard = grafana.dashboard;
4 local row = grafana.row;
5 local singlestat = grafana.singlestat;
6 local prometheus = grafana.prometheus;
7 local graphPanel = grafana.graphPanel;
8 local tablePanel = grafana.tablePanel;
9 local template = grafana.template;
10 {
11 grafanaDashboards+:: {
12 'prometheus.json':
13 local showMultiCluster = $._config.showMultiCluster;
14 local dashboard = g.dashboard(
15 '%(prefix)sOverview' % $._config.grafanaPrometheus
16 );
17 local templatedDashboard = if showMultiCluster then
18 dashboard
19 .addMultiTemplate('cluster', 'prometheus_build_info{%(prometheusSelector)s}' % $._config, $._config.clusterLabel)
20 .addMultiTemplate('job', 'prometheus_build_info{cluster=~"$cluster"}', 'job')
21 .addMultiTemplate('instance', 'prometheus_build_info{cluster=~"$cluster", job=~"$job"}', 'instance')
22 else
23 dashboard
24 .addMultiTemplate('job', 'prometheus_build_info{%(prometheusSelector)s}' % $._config, 'job')
25 .addMultiTemplate('instance', 'prometheus_build_info{job=~"$job"}', 'instance');
26 templatedDashboard
27 .addRow(
28 g.row('Prometheus Stats')
29 .addPanel(
30 g.panel('Prometheus Stats') +
31 g.tablePanel(if showMultiCluster then [
32 'count by (cluster, job, instance, version) (prometheus_build_info{cluster=~"$cluster", job=~"$job", instance=~"$instance"})',
33 'max by (cluster, job, instance) (time() - process_start_time_seconds{cluster=~"$cluster", job=~"$job", instance=~"$instance"})',
34 ] else [
35 'count by (job, instance, version) (prometheus_build_info{job=~"$job", instance=~"$instance"})',
36 'max by (job, instance) (time() - process_start_time_seconds{job=~"$job", instance=~"$instance"})',
37 ], {
38 cluster: { alias: if showMultiCluster then 'Cluster' else '' },
39 job: { alias: 'Job' },
40 instance: { alias: 'Instance' },
41 version: { alias: 'Version' },
42 'Value #A': { alias: 'Count', type: 'hidden' },
43 'Value #B': { alias: 'Uptime', type: 'number', unit: 's' },
44 })
45 )
46 )
47 .addRow(
48 g.row('Discovery')
49 .addPanel(
50 g.panel('Target Sync') +
51 g.queryPanel(if showMultiCluster then 'sum(rate(prometheus_target_sync_length_seconds_sum{cluster=~"$cluster",job=~"$job",instance=~"$instance"}[5m])) by (cluster, job, scrape_job, instance) * 1e3'
52 else 'sum(rate(prometheus_target_sync_length_seconds_sum{job=~"$job",instance=~"$instance"}[5m])) by (scrape_job) * 1e3',
53 if showMultiCluster then '{{cluster}}:{{job}}:{{instance}}:{{scrape_job}}'
54 else '{{scrape_job}}') +
55 { yaxes: g.yaxes('ms') }
56 )
57 .addPanel(
58 g.panel('Targets') +
59 g.queryPanel(if showMultiCluster then 'sum by (cluster, job, instance) (prometheus_sd_discovered_targets{cluster=~"$cluster", job=~"$job",instance=~"$instance"})'
60 else 'sum(prometheus_sd_discovered_targets{job=~"$job",instance=~"$instance"})',
61 if showMultiCluster then '{{cluster}}:{{job}}:{{instance}}'
62 else 'Targets') +
63 g.stack
64 )
65 )
66 .addRow(
67 g.row('Retrieval')
68 .addPanel(
69 g.panel('Average Scrape Interval Duration') +
70 g.queryPanel(if showMultiCluster then 'rate(prometheus_target_interval_length_seconds_sum{cluster=~"$cluster", job=~"$job",instance=~"$instance"}[5m]) / rate(prometheus_target_interval_length_seconds_count{cluster=~"$cluster", job=~"$job",instance=~"$instance"}[5m]) * 1e3'
71 else 'rate(prometheus_target_interval_length_seconds_sum{job=~"$job",instance=~"$instance"}[5m]) / rate(prometheus_target_interval_length_seconds_count{job=~"$job",instance=~"$instance"}[5m]) * 1e3',
72 if showMultiCluster then '{{cluster}}:{{job}}:{{instance}} {{interval}} configured'
73 else '{{interval}} configured') +
74 { yaxes: g.yaxes('ms') }
75 )
76 .addPanel(
77 g.panel('Scrape failures') +
78 g.queryPanel(if showMultiCluster then [
79 'sum by (cluster, job, instance) (rate(prometheus_target_scrapes_exceeded_body_size_limit_total{cluster=~"$cluster",job=~"$job",instance=~"$instance"}[1m]))',
80 'sum by (cluster, job, instance) (rate(prometheus_target_scrapes_exceeded_sample_limit_total{cluster=~"$cluster",job=~"$job",instance=~"$instance"}[1m]))',
81 'sum by (cluster, job, instance) (rate(prometheus_target_scrapes_sample_duplicate_timestamp_total{cluster=~"$cluster",job=~"$job",instance=~"$instance"}[1m]))',
82 'sum by (cluster, job, instance) (rate(prometheus_target_scrapes_sample_out_of_bounds_total{cluster=~"$cluster",job=~"$job",instance=~"$instance"}[1m]))',
83 'sum by (cluster, job, instance) (rate(prometheus_target_scrapes_sample_out_of_order_total{cluster=~"$cluster",job=~"$job",instance=~"$instance"}[1m]))',
84 ] else [
85 'sum by (job) (rate(prometheus_target_scrapes_exceeded_body_size_limit_total[1m]))',
86 'sum by (job) (rate(prometheus_target_scrapes_exceeded_sample_limit_total[1m]))',
87 'sum by (job) (rate(prometheus_target_scrapes_sample_duplicate_timestamp_total[1m]))',
88 'sum by (job) (rate(prometheus_target_scrapes_sample_out_of_bounds_total[1m]))',
89 'sum by (job) (rate(prometheus_target_scrapes_sample_out_of_order_total[1m]))',
90 ], if showMultiCluster then [
91 'exceeded body size limit: {{cluster}} {{job}} {{instance}}',
92 'exceeded sample limit: {{cluster}} {{job}} {{instance}}',
93 'duplicate timestamp: {{cluster}} {{job}} {{instance}}',
94 'out of bounds: {{cluster}} {{job}} {{instance}}',
95 'out of order: {{cluster}} {{job}} {{instance}}',
96 ] else [
97 'exceeded body size limit: {{job}}',
98 'exceeded sample limit: {{job}}',
99 'duplicate timestamp: {{job}}',
100 'out of bounds: {{job}}',
101 'out of order: {{job}}',
102 ]) +
103 g.stack
104 )
105 .addPanel(
106 g.panel('Appended Samples') +
107 g.queryPanel(if showMultiCluster then 'rate(prometheus_tsdb_head_samples_appended_total{cluster=~"$cluster", job=~"$job",instance=~"$instance"}[5m])'
108 else 'rate(prometheus_tsdb_head_samples_appended_total{job=~"$job",instance=~"$instance"}[5m])',
109 if showMultiCluster then '{{cluster}} {{job}} {{instance}}'
110 else '{{job}} {{instance}}') +
111 g.stack
112 )
113 )
114 .addRow(
115 g.row('Storage')
116 .addPanel(
117 g.panel('Head Series') +
118 g.queryPanel(if showMultiCluster then 'prometheus_tsdb_head_series{cluster=~"$cluster",job=~"$job",instance=~"$instance"}'
119 else 'prometheus_tsdb_head_series{job=~"$job",instance=~"$instance"}',
120 if showMultiCluster then '{{cluster}} {{job}} {{instance}} head series'
121 else '{{job}} {{instance}} head series') +
122 g.stack
123 )
124 .addPanel(
125 g.panel('Head Chunks') +
126 g.queryPanel(if showMultiCluster then 'prometheus_tsdb_head_chunks{cluster=~"$cluster",job=~"$job",instance=~"$instance"}'
127 else 'prometheus_tsdb_head_chunks{job=~"$job",instance=~"$instance"}',
128 if showMultiCluster then '{{cluster}} {{job}} {{instance}} head chunks'
129 else '{{job}} {{instance}} head chunks') +
130 g.stack
131 )
132 )
133 .addRow(
134 g.row('Query')
135 .addPanel(
136 g.panel('Query Rate') +
137 g.queryPanel(if showMultiCluster then 'rate(prometheus_engine_query_duration_seconds_count{cluster=~"$cluster",job=~"$job",instance=~"$instance",slice="inner_eval"}[5m])'
138 else 'rate(prometheus_engine_query_duration_seconds_count{job=~"$job",instance=~"$instance",slice="inner_eval"}[5m])',
139 if showMultiCluster then '{{cluster}} {{job}} {{instance}}'
140 else '{{job}} {{instance}}') +
141 g.stack,
142 )
143 .addPanel(
144 g.panel('Stage Duration') +
145 g.queryPanel(if showMultiCluster then 'max by (slice) (prometheus_engine_query_duration_seconds{quantile="0.9",cluster=~"$cluster", job=~"$job",instance=~"$instance"}) * 1e3'
146 else 'max by (slice) (prometheus_engine_query_duration_seconds{quantile="0.9",job=~"$job",instance=~"$instance"}) * 1e3',
147 if showMultiCluster then '{{slice}}'
148 else '{{slice}}') +
149 { yaxes: g.yaxes('ms') } +
150 g.stack,
151 )
152 ) + {
153 tags: $._config.grafanaPrometheus.tags,
154 refresh: $._config.grafanaPrometheus.refresh,
155 },
156 // Remote write specific dashboard.
157 'prometheus-remote-write.json':
158 local timestampComparison =
159 graphPanel.new(
160 'Highest Timestamp In vs. Highest Timestamp Sent',
161 datasource='$datasource',
162 span=6,
163 )
164 .addTarget(prometheus.target(
165 |||
166 (
167 prometheus_remote_storage_highest_timestamp_in_seconds{cluster=~"$cluster", instance=~"$instance"}
168 -
169 ignoring(remote_name, url) group_right(instance) (prometheus_remote_storage_queue_highest_sent_timestamp_seconds{cluster=~"$cluster", instance=~"$instance", url=~"$url"} != 0)
170 )
171 |||,
172 legendFormat='{{cluster}}:{{instance}} {{remote_name}}:{{url}}',
173 ));
174
175 local timestampComparisonRate =
176 graphPanel.new(
177 'Rate[5m]',
178 datasource='$datasource',
179 span=6,
180 )
181 .addTarget(prometheus.target(
182 |||
183 clamp_min(
184 rate(prometheus_remote_storage_highest_timestamp_in_seconds{cluster=~"$cluster", instance=~"$instance"}[5m])
185 -
186 ignoring (remote_name, url) group_right(instance) rate(prometheus_remote_storage_queue_highest_sent_timestamp_seconds{cluster=~"$cluster", instance=~"$instance", url=~"$url"}[5m])
187 , 0)
188 |||,
189 legendFormat='{{cluster}}:{{instance}} {{remote_name}}:{{url}}',
190 ));
191
192 local samplesRate =
193 graphPanel.new(
194 'Rate, in vs. succeeded or dropped [5m]',
195 datasource='$datasource',
196 span=12,
197 )
198 .addTarget(prometheus.target(
199 |||
200 rate(
201 prometheus_remote_storage_samples_in_total{cluster=~"$cluster", instance=~"$instance"}[5m])
202 -
203 ignoring(remote_name, url) group_right(instance) (rate(prometheus_remote_storage_succeeded_samples_total{cluster=~"$cluster", instance=~"$instance", url=~"$url"}[5m]) or rate(prometheus_remote_storage_samples_total{cluster=~"$cluster", instance=~"$instance", url=~"$url"}[5m]))
204 -
205 (rate(prometheus_remote_storage_dropped_samples_total{cluster=~"$cluster", instance=~"$instance", url=~"$url"}[5m]) or rate(prometheus_remote_storage_samples_dropped_total{cluster=~"$cluster", instance=~"$instance", url=~"$url"}[5m]))
206 |||,
207 legendFormat='{{cluster}}:{{instance}} {{remote_name}}:{{url}}'
208 ));
209
210 local currentShards =
211 graphPanel.new(
212 'Current Shards',
213 datasource='$datasource',
214 span=12,
215 min_span=6,
216 )
217 .addTarget(prometheus.target(
218 'prometheus_remote_storage_shards{cluster=~"$cluster", instance=~"$instance", url=~"$url"}',
219 legendFormat='{{cluster}}:{{instance}} {{remote_name}}:{{url}}'
220 ));
221
222 local maxShards =
223 graphPanel.new(
224 'Max Shards',
225 datasource='$datasource',
226 span=4,
227 )
228 .addTarget(prometheus.target(
229 'prometheus_remote_storage_shards_max{cluster=~"$cluster", instance=~"$instance", url=~"$url"}',
230 legendFormat='{{cluster}}:{{instance}} {{remote_name}}:{{url}}'
231 ));
232
233 local minShards =
234 graphPanel.new(
235 'Min Shards',
236 datasource='$datasource',
237 span=4,
238 )
239 .addTarget(prometheus.target(
240 'prometheus_remote_storage_shards_min{cluster=~"$cluster", instance=~"$instance", url=~"$url"}',
241 legendFormat='{{cluster}}:{{instance}} {{remote_name}}:{{url}}'
242 ));
243
244 local desiredShards =
245 graphPanel.new(
246 'Desired Shards',
247 datasource='$datasource',
248 span=4,
249 )
250 .addTarget(prometheus.target(
251 'prometheus_remote_storage_shards_desired{cluster=~"$cluster", instance=~"$instance", url=~"$url"}',
252 legendFormat='{{cluster}}:{{instance}} {{remote_name}}:{{url}}'
253 ));
254
255 local shardsCapacity =
256 graphPanel.new(
257 'Shard Capacity',
258 datasource='$datasource',
259 span=6,
260 )
261 .addTarget(prometheus.target(
262 'prometheus_remote_storage_shard_capacity{cluster=~"$cluster", instance=~"$instance", url=~"$url"}',
263 legendFormat='{{cluster}}:{{instance}} {{remote_name}}:{{url}}'
264 ));
265
266
267 local pendingSamples =
268 graphPanel.new(
269 'Pending Samples',
270 datasource='$datasource',
271 span=6,
272 )
273 .addTarget(prometheus.target(
274 'prometheus_remote_storage_pending_samples{cluster=~"$cluster", instance=~"$instance", url=~"$url"} or prometheus_remote_storage_samples_pending{cluster=~"$cluster", instance=~"$instance", url=~"$url"}',
275 legendFormat='{{cluster}}:{{instance}} {{remote_name}}:{{url}}'
276 ));
277
278 local walSegment =
279 graphPanel.new(
280 'TSDB Current Segment',
281 datasource='$datasource',
282 span=6,
283 formatY1='none',
284 )
285 .addTarget(prometheus.target(
286 'prometheus_tsdb_wal_segment_current{cluster=~"$cluster", instance=~"$instance"}',
287 legendFormat='{{cluster}}:{{instance}}'
288 ));
289
290 local queueSegment =
291 graphPanel.new(
292 'Remote Write Current Segment',
293 datasource='$datasource',
294 span=6,
295 formatY1='none',
296 )
297 .addTarget(prometheus.target(
298 'prometheus_wal_watcher_current_segment{cluster=~"$cluster", instance=~"$instance"}',
299 legendFormat='{{cluster}}:{{instance}} {{consumer}}'
300 ));
301
302 local droppedSamples =
303 graphPanel.new(
304 'Dropped Samples',
305 datasource='$datasource',
306 span=3,
307 )
308 .addTarget(prometheus.target(
309 'rate(prometheus_remote_storage_dropped_samples_total{cluster=~"$cluster", instance=~"$instance", url=~"$url"}[5m]) or rate(prometheus_remote_storage_samples_dropped_total{cluster=~"$cluster", instance=~"$instance", url=~"$url"}[5m])',
310 legendFormat='{{cluster}}:{{instance}} {{remote_name}}:{{url}}'
311 ));
312
313 local failedSamples =
314 graphPanel.new(
315 'Failed Samples',
316 datasource='$datasource',
317 span=3,
318 )
319 .addTarget(prometheus.target(
320 'rate(prometheus_remote_storage_failed_samples_total{cluster=~"$cluster", instance=~"$instance", url=~"$url"}[5m]) or rate(prometheus_remote_storage_samples_failed_total{cluster=~"$cluster", instance=~"$instance", url=~"$url"}[5m])',
321 legendFormat='{{cluster}}:{{instance}} {{remote_name}}:{{url}}'
322 ));
323
324 local retriedSamples =
325 graphPanel.new(
326 'Retried Samples',
327 datasource='$datasource',
328 span=3,
329 )
330 .addTarget(prometheus.target(
331 'rate(prometheus_remote_storage_retried_samples_total{cluster=~"$cluster", instance=~"$instance", url=~"$url"}[5m]) or rate(prometheus_remote_storage_samples_retried_total{cluster=~"$cluster", instance=~"$instance", url=~"$url"}[5m])',
332 legendFormat='{{cluster}}:{{instance}} {{remote_name}}:{{url}}'
333 ));
334
335 local enqueueRetries =
336 graphPanel.new(
337 'Enqueue Retries',
338 datasource='$datasource',
339 span=3,
340 )
341 .addTarget(prometheus.target(
342 'rate(prometheus_remote_storage_enqueue_retries_total{cluster=~"$cluster", instance=~"$instance", url=~"$url"}[5m])',
343 legendFormat='{{cluster}}:{{instance}} {{remote_name}}:{{url}}'
344 ));
345
346 dashboard.new(
347 title='%(prefix)sRemote Write' % $._config.grafanaPrometheus,
348 editable=true
349 )
350 .addTemplate(
351 {
352 hide: 0,
353 label: null,
354 name: 'datasource',
355 options: [],
356 query: 'prometheus',
357 refresh: 1,
358 regex: '',
359 type: 'datasource',
360 },
361 )
362 .addTemplate(
363 template.new(
364 'cluster',
365 '$datasource',
366 'label_values(prometheus_build_info, cluster)' % $._config,
367 refresh='time',
368 current={
369 selected: true,
370 text: 'All',
371 value: '$__all',
372 },
373 includeAll=true,
374 )
375 )
376 .addTemplate(
377 template.new(
378 'instance',
379 '$datasource',
380 'label_values(prometheus_build_info{cluster=~"$cluster"}, instance)' % $._config,
381 refresh='time',
382 current={
383 selected: true,
384 text: 'All',
385 value: '$__all',
386 },
387 includeAll=true,
388 )
389 )
390 .addTemplate(
391 template.new(
392 'url',
393 '$datasource',
394 'label_values(prometheus_remote_storage_shards{cluster=~"$cluster", instance=~"$instance"}, url)' % $._config,
395 refresh='time',
396 includeAll=true,
397 )
398 )
399 .addRow(
400 row.new('Timestamps')
401 .addPanel(timestampComparison)
402 .addPanel(timestampComparisonRate)
403 )
404 .addRow(
405 row.new('Samples')
406 .addPanel(samplesRate)
407 )
408 .addRow(
409 row.new(
410 'Shards'
411 )
412 .addPanel(currentShards)
413 .addPanel(maxShards)
414 .addPanel(minShards)
415 .addPanel(desiredShards)
416 )
417 .addRow(
418 row.new('Shard Details')
419 .addPanel(shardsCapacity)
420 .addPanel(pendingSamples)
421 )
422 .addRow(
423 row.new('Segments')
424 .addPanel(walSegment)
425 .addPanel(queueSegment)
426 )
427 .addRow(
428 row.new('Misc. Rates')
429 .addPanel(droppedSamples)
430 .addPanel(failedSamples)
431 .addPanel(retriedSamples)
432 .addPanel(enqueueRetries)
433 ) + {
434 tags: $._config.grafanaPrometheus.tags,
435 refresh: $._config.grafanaPrometheus.refresh,
436 },
437 },
438 }