Debian Codegraph @

src/shared/watchdog.c

raw
1 /* SPDX-License-Identifier: LGPL-2.1-or-later */
2
3 #include <fcntl.h>
4 #include <linux/watchdog.h>
5 #include <sys/ioctl.h>
6 #include <sys/stat.h>
7 #include <syslog.h>
8 #include <unistd.h>
9
10 #include "sd-messages.h"
11
12 #include "alloc-util.h"
13 #include "devnum-util.h"
14 #include "errno-util.h"
15 #include "fd-util.h"
16 #include "fileio.h"
17 #include "log.h"
18 #include "path-util.h"
19 #include "ratelimit.h"
20 #include "stat-util.h"
21 #include "string-util.h"
22 #include "strv.h"
23 #include "time-util.h"
24 #include "watchdog.h"
25
26 static int watchdog_fd = -EBADF;
27 static char *watchdog_device = NULL;
28 static usec_t watchdog_timeout = 0; /* 0 → close device and USEC_INFINITY → don't change timeout */
29 static usec_t watchdog_pretimeout = 0; /* 0 → disable pretimeout and USEC_INFINITY → don't change pretimeout */
30 static usec_t watchdog_last_good_ping = USEC_INFINITY;
31 static usec_t watchdog_last_bad_ping = USEC_INFINITY;
32 static unsigned watchdog_bad_pings = 0;
33 static bool watchdog_supports_pretimeout = false; /* Depends on kernel state that might change at runtime */
34 static char *watchdog_pretimeout_governor = NULL;
35
36 /* Starting from kernel version 4.5, the maximum allowable watchdog timeout is
37 * UINT_MAX/1000U seconds (since internal calculations are done in milliseconds
38 * using unsigned integers. However, the kernel's userspace API for the watchdog
39 * uses signed integers for its ioctl parameters (even for timeout values and
40 * bit flags) so this is why we must consider the maximum signed integer value
41 * as well.
42 */
43 #define WATCHDOG_TIMEOUT_MAX_SEC (CONST_MIN(UINT_MAX/1000U, (unsigned)INT_MAX))
44
45 /* How many times to ping the watchdog in a single burst. */
46 #define WATCHDOG_PING_BURST 3
47 /* How many times to ping the watchdog in total before giving up on it. */
48 #define WATCHDOG_MAX_FAILED_PINGS 15
49
50 #define WATCHDOG_GOV_NAME_MAXLEN 20 /* From the kernel watchdog driver */
51
52 static int saturated_usec_to_sec(usec_t val) {
53 usec_t t = DIV_ROUND_UP(val, USEC_PER_SEC);
54 return MIN(t, (usec_t) WATCHDOG_TIMEOUT_MAX_SEC); /* Saturate to watchdog max */
55 }
56
57 static int watchdog_get_sysfs_path(const char *filename, char **ret_path) {
58 struct stat st;
59 int r;
60
61 if (watchdog_fd < 0)
62 return -EBADF;
63
64 if (fstat(watchdog_fd, &st))
65 return -errno;
66
67 r = stat_verify_char(&st);
68 if (r < 0)
69 return r;
70
71 if (asprintf(ret_path, "/sys/dev/char/"DEVNUM_FORMAT_STR"/%s", DEVNUM_FORMAT_VAL(st.st_rdev), filename) < 0)
72 return -ENOMEM;
73
74 return 0;
75 }
76
77 static int watchdog_get_pretimeout_governor(char **ret_gov) {
78 _cleanup_free_ char *sys_fn = NULL;
79 int r;
80
81 assert(ret_gov);
82
83 r = watchdog_get_sysfs_path("pretimeout_governor", &sys_fn);
84 if (r < 0)
85 return r;
86
87 log_info("Watchdog: reading from %s", sys_fn);
88
89 r = read_virtual_file(sys_fn, WATCHDOG_GOV_NAME_MAXLEN - 1, ret_gov, NULL);
90 if (r < 0)
91 return r;
92
93 delete_trailing_chars(*ret_gov, WHITESPACE);
94
95 return 0;
96 }
97
98 static int watchdog_set_pretimeout_governor(const char *governor) {
99 _cleanup_free_ char *sys_fn = NULL;
100 int r;
101
102 if (isempty(governor))
103 return 0; /* Nothing to do */
104
105 r = watchdog_get_sysfs_path("pretimeout_governor", &sys_fn);
106 if (r < 0)
107 return r;
108
109 log_info("Watchdog: setting pretimeout_governor to '%s' via '%s'", governor, sys_fn);
110
111 r = write_string_file(sys_fn,
112 governor,
113 WRITE_STRING_FILE_DISABLE_BUFFER | WRITE_STRING_FILE_VERIFY_ON_FAILURE | WRITE_STRING_FILE_VERIFY_IGNORE_NEWLINE);
114 if (r < 0)
115 return log_error_errno(r, "Failed to set watchdog pretimeout_governor to '%s': %m", governor);
116
117 return r;
118 }
119
120 static int watchdog_set_enable(bool enable) {
121 int flags = enable ? WDIOS_ENABLECARD : WDIOS_DISABLECARD;
122
123 assert(watchdog_fd >= 0);
124
125 if (ioctl(watchdog_fd, WDIOC_SETOPTIONS, &flags) < 0) {
126 if (!enable)
127 return log_warning_errno(errno, "Failed to disable hardware watchdog, ignoring: %m");
128
129 /* ENOTTY means the watchdog is always enabled so we're fine */
130 log_full_errno(ERRNO_IS_NOT_SUPPORTED(errno) ? LOG_DEBUG : LOG_WARNING, errno,
131 "Failed to enable hardware watchdog, ignoring: %m");
132 if (!ERRNO_IS_NOT_SUPPORTED(errno))
133 return -errno;
134 }
135
136 return 0;
137 }
138
139 static int watchdog_read_timeout(void) {
140 int sec = 0;
141
142 assert(watchdog_fd >= 0);
143
144 if (ioctl(watchdog_fd, WDIOC_GETTIMEOUT, &sec) < 0)
145 return -errno;
146
147 assert(sec > 0);
148 watchdog_timeout = sec * USEC_PER_SEC;
149
150 return 0;
151 }
152
153 static int watchdog_set_timeout(void) {
154 int sec;
155
156 assert(watchdog_fd >= 0);
157 assert(timestamp_is_set(watchdog_timeout));
158
159 sec = saturated_usec_to_sec(watchdog_timeout);
160
161 if (ioctl(watchdog_fd, WDIOC_SETTIMEOUT, &sec) < 0)
162 return -errno;
163
164 assert(sec > 0); /* buggy driver ? */
165 watchdog_timeout = sec * USEC_PER_SEC;
166
167 return 0;
168 }
169
170 static int watchdog_read_pretimeout(void) {
171 int sec = 0;
172
173 assert(watchdog_fd >= 0);
174
175 if (ioctl(watchdog_fd, WDIOC_GETPRETIMEOUT, &sec) < 0) {
176 watchdog_pretimeout = 0;
177 return log_full_errno(ERRNO_IS_NOT_SUPPORTED(errno) ? LOG_DEBUG : LOG_WARNING, errno, "Failed to get watchdog pretimeout value, ignoring: %m");
178 }
179
180 watchdog_pretimeout = sec * USEC_PER_SEC;
181
182 return 0;
183 }
184
185 static int watchdog_set_pretimeout(void) {
186 int sec;
187
188 assert(watchdog_fd >= 0);
189 assert(watchdog_pretimeout != USEC_INFINITY);
190
191 sec = saturated_usec_to_sec(watchdog_pretimeout);
192
193 if (ioctl(watchdog_fd, WDIOC_SETPRETIMEOUT, &sec) < 0) {
194 watchdog_pretimeout = 0;
195
196 if (ERRNO_IS_NOT_SUPPORTED(errno)) {
197 log_info("Watchdog does not support pretimeouts.");
198 return 0;
199 }
200
201 return log_error_errno(errno, "Failed to set watchdog pretimeout to %s: %m", FORMAT_TIMESPAN(sec, USEC_PER_SEC));
202 }
203
204 /* The set ioctl does not return the actual value set so get it now. */
205 (void) watchdog_read_pretimeout();
206
207 return 0;
208 }
209
210 usec_t watchdog_get_last_ping(clockid_t clock) {
211 return map_clock_usec(watchdog_last_good_ping, CLOCK_BOOTTIME, clock);
212 }
213
214 dual_timestamp* watchdog_get_last_ping_as_dual_timestamp(dual_timestamp *ret) {
215 assert(ret);
216
217 ret->monotonic = watchdog_get_last_ping(CLOCK_MONOTONIC);
218 ret->realtime = watchdog_get_last_ping(CLOCK_REALTIME);
219
220 return ret;
221 }
222
223 static int watchdog_ping_now(void) {
224 int r;
225
226 assert(watchdog_fd >= 0);
227
228 r = RET_NERRNO(ioctl(watchdog_fd, WDIOC_KEEPALIVE, 0));
229 if (r < 0) {
230 watchdog_last_bad_ping = now(CLOCK_BOOTTIME);
231 if (++ watchdog_bad_pings >= WATCHDOG_MAX_FAILED_PINGS) {
232 log_struct_errno(LOG_ERR, r,
233 LOG_MESSAGE("Failed to ping hardware watchdog %s, closing watchdog after %u attempts: %m",
234 watchdog_device, watchdog_bad_pings),
235 "MESSAGE_ID=" SD_MESSAGE_WATCHDOG_PING_FAILED_STR,
236 "WATCHDOG_DEVICE=%s", watchdog_device);
237 watchdog_timeout = USEC_INFINITY;
238 watchdog_close(/* disarm= */ true);
239 } else
240 log_struct_errno(LOG_WARNING, r,
241 LOG_MESSAGE("Failed to ping hardware watchdog %s: %m",
242 watchdog_device),
243 "MESSAGE_ID=" SD_MESSAGE_WATCHDOG_PING_FAILED_STR,
244 "WATCHDOG_DEVICE=%s", watchdog_device);
245 } else {
246 watchdog_last_good_ping = now(CLOCK_BOOTTIME);
247 if (watchdog_bad_pings > 0) {
248 log_struct(LOG_INFO,
249 LOG_MESSAGE("Successfully pinged hardware watchdog %s after %u attempts: %m",
250 watchdog_device, watchdog_bad_pings + 1),
251 "WATCHDOG_DEVICE=%s", watchdog_device);
252 watchdog_bad_pings = 0;
253 watchdog_last_bad_ping = USEC_INFINITY;
254 }
255 }
256
257 return r;
258 }
259
260 static int watchdog_update_pretimeout(void) {
261 _cleanup_free_ char *governor = NULL;
262 int r, t_sec, pt_sec;
263
264 if (watchdog_fd < 0)
265 return 0;
266
267 if (watchdog_timeout == USEC_INFINITY || watchdog_pretimeout == USEC_INFINITY)
268 return 0;
269
270 if (!watchdog_supports_pretimeout && watchdog_pretimeout == 0)
271 return 0; /* Nothing to do */
272
273 /* The configuration changed, do not assume it can still work, as the module(s)
274 * might have been unloaded. */
275 watchdog_supports_pretimeout = false;
276
277 /* Update the pretimeout governor as well */
278 (void) watchdog_set_pretimeout_governor(watchdog_pretimeout_governor);
279
280 r = watchdog_get_pretimeout_governor(&governor);
281 if (r < 0)
282 return log_warning_errno(r, "Watchdog: failed to read pretimeout governor: %m");
283 if (isempty(governor))
284 return log_warning_errno(SYNTHETIC_ERRNO(EINVAL),
285 "Watchdog: no pretimeout governor detected - is the required kernel module loaded?");
286
287 /* If we have a pretimeout governor, then pretimeout is supported. Without a governor
288 * pretimeout does not work at all.
289 * Note that this might require a kernel module that is not autoloaded, so we don't
290 * cache this, but we check every time the configuration changes. */
291 watchdog_supports_pretimeout = true;
292
293 /* Determine if the pretimeout is valid for the current watchdog timeout. */
294 t_sec = saturated_usec_to_sec(watchdog_timeout);
295 pt_sec = saturated_usec_to_sec(watchdog_pretimeout);
296 if (pt_sec >= t_sec) {
297 r = log_error_errno(SYNTHETIC_ERRNO(EINVAL),
298 "Cannot set watchdog pretimeout to %is (%s watchdog timeout of %is)",
299 pt_sec, pt_sec == t_sec ? "same as" : "longer than", t_sec);
300 (void) watchdog_read_pretimeout();
301 } else
302 r = watchdog_set_pretimeout();
303
304 if (watchdog_pretimeout == 0)
305 log_info("Watchdog pretimeout is disabled.");
306 else
307 log_info("Watchdog running with a pretimeout of %s with governor '%s'.",
308 FORMAT_TIMESPAN(watchdog_pretimeout, 0),
309 governor);
310
311 return r;
312 }
313
314 static int watchdog_update_timeout(void) {
315 int r;
316 usec_t previous_timeout;
317
318 assert(watchdog_timeout > 0);
319
320 if (watchdog_fd < 0)
321 return 0;
322
323 previous_timeout = watchdog_timeout;
324
325 if (watchdog_timeout != USEC_INFINITY) {
326 r = watchdog_set_timeout();
327 if (r < 0) {
328 if (!ERRNO_IS_NOT_SUPPORTED(r))
329 return log_error_errno(r, "Failed to set watchdog hardware timeout to %s: %m",
330 FORMAT_TIMESPAN(watchdog_timeout, 0));
331
332 log_info("Modifying watchdog hardware timeout is not supported, reusing the programmed timeout.");
333 watchdog_timeout = USEC_INFINITY;
334 }
335 }
336
337 if (watchdog_timeout == USEC_INFINITY) {
338 r = watchdog_read_timeout();
339 if (r < 0) {
340 if (!ERRNO_IS_NOT_SUPPORTED(r))
341 return log_error_errno(r, "Failed to query watchdog hardware timeout: %m");
342 log_info("Reading watchdog hardware timeout is not supported, reusing the configured timeout.");
343 watchdog_timeout = previous_timeout;
344 }
345 }
346
347 /* If the watchdog timeout was changed, the pretimeout could have been
348 * changed as well by the driver or the kernel so we need to update the
349 * pretimeout now. Or if the watchdog is being configured for the first
350 * time, we want to configure the pretimeout before it is enabled. */
351 (void) watchdog_update_pretimeout();
352
353 r = watchdog_set_enable(true);
354 if (r < 0)
355 return r;
356
357 log_info("Watchdog running with a hardware timeout of %s.", FORMAT_TIMESPAN(watchdog_timeout, 0));
358
359 return watchdog_ping_now();
360 }
361
362 static int watchdog_open(bool ignore_ratelimit) {
363 static RateLimit watchdog_open_ratelimit = { 0, 1 }; /* The interval is initialized dynamically below. */
364 struct watchdog_info ident;
365 char **try_order;
366 int r;
367
368 assert(watchdog_fd < 0);
369 assert(watchdog_timeout > 0);
370
371 watchdog_open_ratelimit.interval = MIN(watchdog_timeout / 2, 5 * USEC_PER_SEC);
372
373 if (!ratelimit_below(&watchdog_open_ratelimit))
374 if (!ignore_ratelimit) /* If ignore_ratelimit, we update the timestamp, but continue. */
375 return -EWOULDBLOCK;
376
377 /* Let's prefer new-style /dev/watchdog0 (i.e. kernel 3.5+) over classic /dev/watchdog. The former
378 * has the benefit that we can easily find the matching directory in sysfs from it, as the relevant
379 * sysfs attributes can only be found via /sys/dev/char/<major>:<minor> if the new-style device
380 * major/minor is used, not the old-style. */
381 try_order = !watchdog_device || PATH_IN_SET(watchdog_device, "/dev/watchdog", "/dev/watchdog0") ?
382 STRV_MAKE("/dev/watchdog0", "/dev/watchdog") : STRV_MAKE(watchdog_device);
383
384 STRV_FOREACH(wd, try_order) {
385 watchdog_fd = RET_NERRNO(open(*wd, O_WRONLY|O_CLOEXEC));
386 if (watchdog_fd >= 0) {
387 if (free_and_strdup(&watchdog_device, *wd) < 0) {
388 r = log_oom_debug();
389 goto close_and_fail;
390 }
391
392 break;
393 }
394
395 if (watchdog_fd != -ENOENT)
396 return log_struct_errno(LOG_WARNING, watchdog_fd,
397 LOG_MESSAGE("Failed to open watchdog device %s: %m", *wd),
398 "MESSAGE_ID=" SD_MESSAGE_WATCHDOG_OPEN_FAILED_STR,
399 "WATCHDOG_DEVICE=%s", *wd);
400 }
401
402 if (watchdog_fd < 0)
403 return log_debug_errno(watchdog_fd, "Failed to open %swatchdog device%s%s.",
404 watchdog_device ? "" : "any ",
405 watchdog_device ? " " : "",
406 strempty(watchdog_device));
407
408 watchdog_last_good_ping = watchdog_last_bad_ping = USEC_INFINITY;
409 watchdog_bad_pings = 0;
410
411 r = RET_NERRNO(ioctl(watchdog_fd, WDIOC_GETSUPPORT, &ident));
412 if (r < 0)
413 log_struct_errno(LOG_INFO, r,
414 LOG_MESSAGE("Using hardware watchdog %s, no support for WDIOC_GETSUPPORT ioctl: %m",
415 watchdog_device),
416 "MESSAGE_ID=" SD_MESSAGE_WATCHDOG_OPENED_STR,
417 "WATCHDOG_DEVICE=%s", watchdog_device);
418 else
419 log_struct(LOG_INFO,
420 LOG_MESSAGE("Using hardware watchdog %s: '%s', version %x.",
421 watchdog_device,
422 ident.identity,
423 ident.firmware_version),
424 "MESSAGE_ID=" SD_MESSAGE_WATCHDOG_OPENED_STR,
425 "WATCHDOG_DEVICE=%s", watchdog_device);
426
427 r = watchdog_update_timeout();
428 if (r < 0)
429 goto close_and_fail;
430
431 return 0;
432
433 close_and_fail:
434 watchdog_close(/* disarm= */ true);
435 return r;
436 }
437
438 const char* watchdog_get_device(void) {
439 return watchdog_device;
440 }
441
442 int watchdog_set_device(const char *path) {
443 int r;
444
445 r = free_and_strdup(&watchdog_device, path);
446 if (r > 0) /* watchdog_device changed */
447 watchdog_close(/* disarm= */ true);
448
449 return r;
450 }
451
452 int watchdog_setup(usec_t timeout) {
453 usec_t previous_timeout;
454 int r;
455
456 /* timeout=0 closes the device whereas passing timeout=USEC_INFINITY opens it (if needed)
457 * without configuring any particular timeout and thus reuses the programmed value (therefore
458 * it's a nop if the device is already opened). */
459
460 if (timeout == 0) {
461 watchdog_close(/* disarm= */ true);
462 return 0;
463 }
464
465 /* Let's shortcut duplicated requests */
466 if (watchdog_fd >= 0 && (timeout == watchdog_timeout || timeout == USEC_INFINITY))
467 return 0;
468
469 /* Initialize the watchdog timeout with the caller value. This value is going to be updated by
470 * update_timeout() with the closest value supported by the driver */
471 previous_timeout = watchdog_timeout;
472 watchdog_timeout = timeout;
473
474 if (watchdog_fd < 0)
475 return watchdog_open(/* ignore_ratelimit= */ false);
476
477 r = watchdog_update_timeout();
478 if (r < 0)
479 watchdog_timeout = previous_timeout;
480
481 return r;
482 }
483
484 int watchdog_setup_pretimeout(usec_t timeout) {
485 /* timeout=0 disables the pretimeout whereas timeout=USEC_INFINITY is a nop. */
486 if ((watchdog_fd >= 0 && timeout == watchdog_pretimeout) || timeout == USEC_INFINITY)
487 return 0;
488
489 /* Initialize the watchdog timeout with the caller value. This value is
490 * going to be updated by update_pretimeout() with the running value,
491 * even if it fails to update the timeout. */
492 watchdog_pretimeout = timeout;
493
494 return watchdog_update_pretimeout();
495 }
496
497 int watchdog_setup_pretimeout_governor(const char *governor) {
498 if (free_and_strdup(&watchdog_pretimeout_governor, governor) < 0)
499 return -ENOMEM;
500
501 return watchdog_set_pretimeout_governor(watchdog_pretimeout_governor);
502 }
503
504 static usec_t watchdog_calc_timeout(void) {
505 /* If we failed to many times, don't schedule the next attempt at all. */
506 if (watchdog_bad_pings >= WATCHDOG_MAX_FAILED_PINGS)
507 return 0;
508
509 /* Calculate the effective timeout which accounts for the watchdog
510 * pretimeout if configured and supported. */
511 if (watchdog_supports_pretimeout && timestamp_is_set(watchdog_pretimeout) && watchdog_timeout >= watchdog_pretimeout)
512 return watchdog_timeout - watchdog_pretimeout;
513 else
514 return watchdog_timeout;
515 }
516
517 usec_t watchdog_runtime_wait(unsigned divisor) {
518 usec_t timeout = watchdog_calc_timeout();
519 if (!timestamp_is_set(timeout))
520 return USEC_INFINITY;
521
522 /* Sleep watchdog timeout / divisor since the last successful ping attempt at most,
523 * or the fifth of that if the ping failed. */
524
525 usec_t ts = MAX(timestamp_is_set(watchdog_last_good_ping) ? watchdog_last_good_ping : 0u,
526 timestamp_is_set(watchdog_last_bad_ping) ? watchdog_last_bad_ping : 0u);
527 if (ts > 0) {
528 usec_t ntime = now(CLOCK_BOOTTIME);
529 if (ts == watchdog_last_bad_ping)
530 divisor *= 5;
531
532 assert(ntime >= ts);
533 return usec_sub_unsigned(ts + timeout / divisor, ntime);
534 }
535
536 return timeout / divisor;
537 }
538
539 int watchdog_ping(void) {
540 if (watchdog_timeout == 0)
541 return 0;
542
543 if (watchdog_fd < 0)
544 /* open_watchdog() will automatically ping the device for us if necessary */
545 return watchdog_open(/* ignore_ratelimit= */ false);
546
547 /* Ping approximately watchdog_timeout/4 after a successful ping, or even less than that
548 * after an unsuccessful ping. */
549 if (watchdog_runtime_wait(/* divisor= */ 4) > 0)
550 return 0;
551
552 return watchdog_ping_now();
553 }
554
555 void watchdog_report_if_missing(void) {
556 /* If we failed to open the watchdog because the device doesn't exist, report why. If we successfully
557 * opened a device, or opening failed for a different reason, we logged then. But ENOENT failure may
558 * be transient, for example because modules being loaded or the hardware being detected. This
559 * function can be called to log after things have settled down.
560 *
561 * If a device was specified explicitly, raise level. */
562
563 if (watchdog_fd != -ENOENT)
564 return;
565
566 /* We attempt to open the watchdog one last time here, so that if we log, we log accurately. */
567 if (watchdog_open(/* ignore_ratelimit= */ true) >= 0)
568 return;
569
570 if (watchdog_device)
571 log_struct_errno(LOG_WARNING, watchdog_fd,
572 LOG_MESSAGE("Failed to open watchdog device %s before the initial transaction completed: %m",
573 watchdog_device),
574 "MESSAGE_ID=" SD_MESSAGE_WATCHDOG_OPEN_FAILED_STR,
575 "WATCHDOG_DEVICE=%s", watchdog_device);
576 else
577 log_struct_errno(LOG_WARNING, watchdog_fd,
578 LOG_MESSAGE("Failed to open any watchdog device before the initial transaction completed: %m"),
579 "MESSAGE_ID=" SD_MESSAGE_WATCHDOG_OPEN_FAILED_STR);
580 }
581
582 void watchdog_close(bool disarm) {
583
584 /* Once closed, pinging the device becomes a NOP and we request a new
585 * call to watchdog_setup() to open the device again. */
586 watchdog_timeout = 0;
587
588 if (watchdog_fd < 0)
589 return;
590
591 if (disarm) {
592 (void) watchdog_set_enable(false);
593
594 /* To be sure, use magic close logic, too */
595 for (;;) {
596 static const char v = 'V';
597
598 if (write(watchdog_fd, &v, 1) > 0)
599 break;
600
601 if (errno != EINTR) {
602 log_warning_errno(errno, "Failed to disarm watchdog timer, ignoring: %m");
603 break;
604 }
605 }
606 }
607
608 watchdog_fd = safe_close(watchdog_fd);
609 }