Debian Codegraph @

src/shared/copy.c

raw
1 /* SPDX-License-Identifier: LGPL-2.1-or-later */
2
3 #include <fcntl.h>
4 #include <linux/btrfs.h>
5 #include <linux/fsverity.h>
6 #include <stddef.h>
7 #include <stdio.h>
8 #include <sys/file.h>
9 #include <sys/ioctl.h>
10 #include <sys/sendfile.h>
11 #include <sys/sysmacros.h>
12 #include <unistd.h>
13
14 #include "alloc-util.h"
15 #include "btrfs-util.h"
16 #include "chattr-util.h"
17 #include "copy.h"
18 #include "dirent-util.h"
19 #include "errno-util.h"
20 #include "fd-util.h"
21 #include "fileio.h"
22 #include "fs-util.h"
23 #include "hashmap.h"
24 #include "log.h"
25 #include "mountpoint-util.h"
26 #include "nulstr-util.h"
27 #include "path-util.h"
28 #include "rm-rf.h"
29 #include "selinux-util.h"
30 #include "signal-util.h"
31 #include "stat-util.h"
32 #include "stdio-util.h"
33 #include "string-util.h"
34 #include "sync-util.h"
35 #include "time-util.h"
36 #include "tmpfile-util.h"
37 #include "umask-util.h"
38 #include "user-util.h"
39 #include "xattr-util.h"
40
41 /* If we copy via a userspace buffer, size it to 64K */
42 #define COPY_BUFFER_SIZE (64U*U64_KB)
43
44 /* If a byte progress function is specified during copying, never try to copy more than 1M, so that we can
45 * reasonably call the progress function still */
46 #define PROGRESS_STEP_SIZE (1U*U64_MB)
47
48 /* A safety net for descending recursively into file system trees to copy. On Linux PATH_MAX is 4096, which means the
49 * deepest valid path one can build is around 2048, which we hence use as a safety net here, to not spin endlessly in
50 * case of bind mount cycles and suchlike. */
51 #define COPY_DEPTH_MAX 2048U
52
53 static ssize_t try_copy_file_range(
54 int fd_in, loff_t *off_in,
55 int fd_out, loff_t *off_out,
56 size_t len,
57 unsigned flags) {
58
59 static int have = -1;
60 ssize_t r;
61
62 if (have == 0)
63 return -ENOSYS;
64
65 r = copy_file_range(fd_in, off_in, fd_out, off_out, len, flags);
66 if (have < 0)
67 have = r >= 0 || errno != ENOSYS;
68 if (r < 0)
69 return -errno;
70
71 return r;
72 }
73
74 static int look_for_signals(CopyFlags copy_flags) {
75 int r;
76
77 if ((copy_flags & (COPY_SIGINT|COPY_SIGTERM)) == 0)
78 return 0;
79
80 r = pop_pending_signal(copy_flags & COPY_SIGINT ? SIGINT : 0,
81 copy_flags & COPY_SIGTERM ? SIGTERM : 0);
82 if (r < 0)
83 return r;
84 if (r != 0)
85 return log_debug_errno(SYNTHETIC_ERRNO(EINTR),
86 "Got %s, cancelling copy operation.", signal_to_string(r));
87
88 return 0;
89 }
90
91 static int create_hole(int fd, off_t size) {
92 off_t offset;
93 off_t end;
94
95 offset = lseek(fd, 0, SEEK_CUR);
96 if (offset < 0)
97 return -errno;
98
99 end = lseek(fd, 0, SEEK_END);
100 if (end < 0)
101 return -errno;
102
103 /* If we're not at the end of the target file, try to punch a hole in the existing space using fallocate(). */
104
105 if (offset < end &&
106 fallocate(fd, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, offset, MIN(size, end - offset)) < 0 &&
107 !ERRNO_IS_NOT_SUPPORTED(errno))
108 return -errno;
109
110 if (end - offset >= size) {
111 /* If we've created the full hole, set the file pointer to the end of the hole we created and exit. */
112 if (lseek(fd, offset + size, SEEK_SET) < 0)
113 return -errno;
114
115 return 0;
116 }
117
118 /* If we haven't created the full hole, use ftruncate() to grow the file (and the hole) to the
119 * required size and move the file pointer to the end of the file. */
120
121 size -= end - offset;
122
123 if (ftruncate(fd, end + size) < 0)
124 return -errno;
125
126 if (lseek(fd, 0, SEEK_END) < 0)
127 return -errno;
128
129 return 0;
130 }
131
132 int copy_bytes_full(
133 int fdf, int fdt,
134 uint64_t max_bytes,
135 CopyFlags copy_flags,
136 void **ret_remains,
137 size_t *ret_remains_size,
138 copy_progress_bytes_t progress,
139 void *userdata) {
140
141 _cleanup_close_ int fdf_opened = -EBADF, fdt_opened = -EBADF;
142 bool try_cfr = true, try_sendfile = true;
143 uint64_t copied_total = 0;
144 int r;
145
146 assert(fdf >= 0);
147 assert(fdt >= 0);
148 assert(!FLAGS_SET(copy_flags, COPY_LOCK_BSD));
149
150 /* Tries to copy bytes from the file descriptor 'fdf' to 'fdt' in the smartest possible way. Copies a
151 * maximum of 'max_bytes', which may be specified as UINT64_MAX, in which no maximum is applied.
152 * Returns negative on error, zero if EOF is hit before the bytes limit is hit and positive
153 * otherwise. If the copy fails for some reason but we read but didn't yet write some data and
154 * ret_remains/ret_remains_size is not NULL, then it will be initialized with an allocated buffer
155 * containing this "remaining" data. Note that these two parameters are initialized with a valid
156 * buffer only on failure and only if there's actually data already read. Otherwise these parameters
157 * if non-NULL are set to NULL. */
158
159 if (ret_remains)
160 *ret_remains = NULL;
161 if (ret_remains_size)
162 *ret_remains_size = 0;
163
164 fdf = fd_reopen_condition(fdf, O_CLOEXEC | O_NOCTTY | O_RDONLY, O_PATH, &fdf_opened);
165 if (fdf < 0)
166 return fdf;
167 fdt = fd_reopen_condition(fdt, O_CLOEXEC | O_NOCTTY | O_RDWR, O_PATH, &fdt_opened);
168 if (fdt < 0)
169 return fdt;
170
171 if (FLAGS_SET(copy_flags, COPY_SEEK0_SOURCE) &&
172 lseek(fdf, 0, SEEK_SET) < 0)
173 return -errno;
174
175 if (FLAGS_SET(copy_flags, COPY_SEEK0_TARGET) &&
176 lseek(fdt, 0, SEEK_SET) < 0)
177 return -errno;
178
179 /* Try btrfs reflinks first. This only works on regular, seekable files, hence let's check the file offsets of
180 * source and destination first. */
181 if ((copy_flags & COPY_REFLINK)) {
182 off_t foffset;
183
184 /* In reflink mode, we need to know the current file offset, unless we already sought to 0 anyway. */
185 foffset = FLAGS_SET(copy_flags, COPY_SEEK0_SOURCE) ? 0 : lseek(fdf, 0, SEEK_CUR);
186 if (foffset >= 0) {
187 off_t toffset;
188
189 toffset = FLAGS_SET(copy_flags, COPY_SEEK0_TARGET) ? 0 : lseek(fdt, 0, SEEK_CUR);
190 if (toffset >= 0) {
191
192 if (foffset == 0 && toffset == 0 && max_bytes == UINT64_MAX)
193 r = reflink(fdf, fdt); /* full file reflink */
194 else
195 r = reflink_range(fdf, foffset, fdt, toffset, max_bytes == UINT64_MAX ? 0 : max_bytes); /* partial reflink */
196 if (r >= 0) {
197 off_t t;
198 int ret;
199
200 /* This worked, yay! Now — to be fully correct — let's adjust the file pointers */
201 if (max_bytes == UINT64_MAX) {
202
203 /* We cloned to the end of the source file, let's position the read
204 * pointer there, and query it at the same time. */
205 t = lseek(fdf, 0, SEEK_END);
206 if (t < 0)
207 return -errno;
208 if (t < foffset)
209 return -ESPIPE;
210
211 /* Let's adjust the destination file write pointer by the same number
212 * of bytes. */
213 t = lseek(fdt, toffset + (t - foffset), SEEK_SET);
214 if (t < 0)
215 return -errno;
216
217 if (FLAGS_SET(copy_flags, COPY_VERIFY_LINKED)) {
218 r = fd_verify_linked(fdf);
219 if (r < 0)
220 return r;
221 }
222
223 /* We copied the whole thing, hence hit EOF, return 0. */
224 ret = 0;
225 } else {
226 t = lseek(fdf, foffset + max_bytes, SEEK_SET);
227 if (t < 0)
228 return -errno;
229
230 t = lseek(fdt, toffset + max_bytes, SEEK_SET);
231 if (t < 0)
232 return -errno;
233
234 /* We copied only some number of bytes, which worked, but
235 * this means we didn't hit EOF, return 1. */
236 ret = 1;
237 }
238
239 if (FLAGS_SET(copy_flags, COPY_VERIFY_LINKED)) {
240 r = fd_verify_linked(fdf);
241 if (r < 0)
242 return r;
243 }
244
245 return ret;
246 }
247 }
248 }
249 }
250
251 usec_t start_timestamp = USEC_INFINITY;
252 if (progress)
253 start_timestamp = now(CLOCK_MONOTONIC);
254
255 for (;;) {
256 ssize_t n;
257 size_t m;
258
259 if (max_bytes <= 0)
260 break;
261
262 r = look_for_signals(copy_flags);
263 if (r < 0)
264 return r;
265
266 /* sendfile() accepts at most SSIZE_MAX-offset bytes to copy, hence let's subtract how much
267 * copied so far from SSIZE_MAX as maximum of what we want to copy. */
268 if (try_sendfile) {
269 assert(copied_total < SSIZE_MAX);
270 m = (uint64_t) SSIZE_MAX - copied_total;
271 } else
272 m = SSIZE_MAX;
273
274 if (max_bytes != UINT64_MAX && m > max_bytes)
275 m = max_bytes;
276
277 if (progress && m > PROGRESS_STEP_SIZE)
278 m = PROGRESS_STEP_SIZE;
279
280 if (copy_flags & COPY_HOLES) {
281 off_t c, e;
282
283 c = lseek(fdf, 0, SEEK_CUR);
284 if (c < 0)
285 return -errno;
286
287 /* To see if we're in a hole, we search for the next data offset. */
288 e = lseek(fdf, c, SEEK_DATA);
289 if (e < 0 && errno == ENXIO)
290 /* If errno == ENXIO, that means we've reached the final hole of the file and
291 * that hole isn't followed by more data. */
292 e = lseek(fdf, 0, SEEK_END);
293 if (e < 0)
294 return -errno;
295
296 /* If we're in a hole (current offset is not a data offset), create a hole of the
297 * same size in the target file. */
298 if (e > c) {
299 /* Make sure our new hole doesn't go over the maximum size we're allowed to copy. */
300 n = MIN(max_bytes, (uint64_t) e - c);
301 r = create_hole(fdt, n);
302 if (r < 0)
303 return r;
304
305 /* Make sure holes are taken into account in the maximum size we're supposed to copy. */
306 if (max_bytes != UINT64_MAX) {
307 max_bytes -= n;
308 if (max_bytes <= 0)
309 break;
310 }
311
312 /* Update the size we're supposed to copy in this iteration if needed. */
313 if (m > max_bytes)
314 m = max_bytes;
315 }
316
317 c = e; /* Set c to the start of the data segment. */
318
319 /* After copying a potential hole, find the end of the data segment by looking for
320 * the next hole. If we get ENXIO, we're at EOF. */
321 e = lseek(fdf, c, SEEK_HOLE);
322 if (e < 0) {
323 if (errno == ENXIO)
324 break;
325 return -errno;
326 }
327 if (e == c)
328 /* Empty data segment? Maybe concurrent hole punching taking place? Or EOF?
329 * Let's figure this out by copying the smallest amount possible */
330 m = 1;
331 else {
332 assert(e > c);
333
334 /* SEEK_HOLE modifies the file offset so we need to move back to the initial offset. */
335 if (lseek(fdf, c, SEEK_SET) < 0)
336 return -errno;
337
338 /* Make sure we're not copying more than the current data segment. */
339 m = MIN(m, (size_t) e - c);
340 assert(m > 0);
341 }
342 }
343
344 /* First, try copy_file_range(), unless we already tried */
345 if (try_cfr) {
346 n = try_copy_file_range(fdf, NULL, fdt, NULL, m, 0u);
347 if (n < 0) {
348 if (!IN_SET(n, -EINVAL, -ENOSYS, -EXDEV, -EBADF, -EOPNOTSUPP))
349 return n;
350
351 try_cfr = false;
352 /* use fallback below */
353 } else if (n == 0) { /* likely EOF */
354
355 if (copied_total > 0)
356 break;
357
358 /* So, we hit EOF immediately, without having copied a single byte. This
359 * could indicate two things: the file is actually empty, or we are on some
360 * virtual file system such as procfs/sysfs where the syscall actually
361 * doesn't work but doesn't return an error. Try to handle that, by falling
362 * back to simple read()s in case we encounter empty files.
363 *
364 * See: https://lwn.net/Articles/846403/ */
365 try_cfr = try_sendfile = false;
366 } else
367 /* Success! */
368 goto next;
369 }
370
371 /* Second, try sendfile(), unless we already tried */
372 if (try_sendfile) {
373 n = sendfile(fdt, fdf, NULL, m);
374 if (n < 0) {
375 if (!IN_SET(errno, EINVAL, ENOSYS))
376 return -errno;
377
378 try_sendfile = false;
379 /* use fallback below */
380 } else if (n == 0) { /* likely EOF */
381
382 if (copied_total > 0)
383 break;
384
385 try_sendfile = false; /* same logic as above for copy_file_range() */
386 } else
387 /* Success! */
388 goto next;
389 }
390
391 /* Third, as a fallback just copy bits by hand */
392 {
393 uint8_t buf[MIN(m, COPY_BUFFER_SIZE)], *p = buf;
394 ssize_t z;
395
396 n = read(fdf, buf, sizeof buf);
397 if (n < 0)
398 return -errno;
399 if (n == 0) /* EOF */
400 break;
401
402 z = (size_t) n;
403 do {
404 ssize_t k;
405
406 k = write(fdt, p, z);
407 if (k < 0) {
408 r = -errno;
409
410 if (ret_remains) {
411 void *copy;
412
413 copy = memdup(p, z);
414 if (!copy)
415 return -ENOMEM;
416
417 *ret_remains = copy;
418 }
419
420 if (ret_remains_size)
421 *ret_remains_size = z;
422
423 return r;
424 }
425
426 assert(k <= z);
427 z -= k;
428 p += k;
429 } while (z > 0);
430 }
431
432 next:
433 copied_total += n;
434
435 /* Disable sendfile() in case we are getting too close to it's SSIZE_MAX-offset limit */
436 if (copied_total > SSIZE_MAX - COPY_BUFFER_SIZE)
437 try_sendfile = false;
438
439 if (progress) {
440 usec_t t = now(CLOCK_MONOTONIC);
441 usec_t d = usec_sub_unsigned(t, start_timestamp);
442 uint64_t bps = UINT64_MAX;
443 if (d > USEC_PER_SEC * 3U)
444 bps = (uint64_t) (copied_total / ((double) d / USEC_PER_SEC));
445
446 r = progress(n, bps, userdata);
447 if (r < 0)
448 return r;
449 }
450
451 if (max_bytes != UINT64_MAX) {
452 assert(max_bytes >= (uint64_t) n);
453 max_bytes -= n;
454 }
455 }
456
457 if (FLAGS_SET(copy_flags, COPY_VERIFY_LINKED)) {
458 r = fd_verify_linked(fdf);
459 if (r < 0)
460 return r;
461 }
462
463 if (copy_flags & COPY_TRUNCATE) {
464 off_t off = lseek(fdt, 0, SEEK_CUR);
465 if (off < 0)
466 return -errno;
467
468 if (ftruncate(fdt, off) < 0)
469 return -errno;
470 }
471
472 return max_bytes <= 0; /* return 0 if we hit EOF earlier than the size limit */
473 }
474
475 static int fd_copy_symlink(
476 int df,
477 const char *from,
478 const struct stat *st,
479 int dt,
480 const char *to,
481 uid_t override_uid,
482 gid_t override_gid,
483 CopyFlags copy_flags) {
484
485 _cleanup_free_ char *target = NULL;
486 int r;
487
488 assert(st);
489 assert(to);
490
491 r = readlinkat_malloc(df, from, &target);
492 if (r < 0)
493 return r;
494
495 if (copy_flags & COPY_MAC_CREATE) {
496 r = mac_selinux_create_file_prepare_at(dt, to, S_IFLNK);
497 if (r < 0)
498 return r;
499 }
500 r = RET_NERRNO(symlinkat(target, dt, to));
501 if (copy_flags & COPY_MAC_CREATE)
502 mac_selinux_create_file_clear();
503 if (r < 0) {
504 if (FLAGS_SET(copy_flags, COPY_GRACEFUL_WARN) && (ERRNO_IS_PRIVILEGE(r) || ERRNO_IS_NOT_SUPPORTED(r))) {
505 log_notice_errno(r, "Failed to copy symlink%s%s%s, ignoring: %m",
506 isempty(from) ? "" : " '",
507 strempty(from),
508 isempty(from) ? "" : "'");
509 return 0;
510 }
511
512 return r;
513 }
514
515 if (fchownat(dt, to,
516 uid_is_valid(override_uid) ? override_uid : st->st_uid,
517 gid_is_valid(override_gid) ? override_gid : st->st_gid,
518 AT_SYMLINK_NOFOLLOW) < 0)
519 r = -errno;
520
521 (void) copy_xattr(df, from, dt, to, copy_flags);
522 (void) utimensat(dt, to, (struct timespec[]) { st->st_atim, st->st_mtim }, AT_SYMLINK_NOFOLLOW);
523 return r;
524 }
525
526 /* Encapsulates the database we store potential hardlink targets in */
527 typedef struct HardlinkContext {
528 int dir_fd; /* An fd to the directory we use as lookup table. Never AT_FDCWD. Lazily created, when
529 * we add the first entry. */
530
531 /* These two fields are used to create the hardlink repository directory above — via
532 * mkdirat(parent_fd, subdir) — and are kept so that we can automatically remove the directory again
533 * when we are done. */
534 int parent_fd; /* Possibly AT_FDCWD */
535 char *subdir;
536 } HardlinkContext;
537
538 static int hardlink_context_setup(
539 HardlinkContext *c,
540 int dt,
541 const char *to,
542 CopyFlags copy_flags) {
543
544 _cleanup_close_ int dt_copy = -EBADF;
545 int r;
546
547 assert(c);
548 assert(c->dir_fd < 0 && c->dir_fd != AT_FDCWD);
549 assert(c->parent_fd < 0);
550 assert(!c->subdir);
551
552 /* If hardlink recreation is requested we have to maintain a database of inodes that are potential
553 * hardlink sources. Given that generally disk sizes have to be assumed to be larger than what fits
554 * into physical RAM we cannot maintain that database in dynamic memory alone. Here we opt to
555 * maintain it on disk, to simplify things: inside the destination directory we'll maintain a
556 * temporary directory consisting of hardlinks of every inode we copied that might be subject of
557 * hardlinks. We can then use that as hardlink source later on. Yes, this means additional disk IO
558 * but thankfully Linux is optimized for this kind of thing. If this ever becomes a performance
559 * bottleneck we can certainly place an in-memory hash table in front of this, but for the beginning,
560 * let's keep things simple, and just use the disk as lookup table for inodes.
561 *
562 * Note that this should have zero performance impact as long as .n_link of all files copied remains
563 * <= 0, because in that case we will not actually allocate the hardlink inode lookup table directory
564 * on disk (we do so lazily, when the first candidate with .n_link > 1 is seen). This means, in the
565 * common case where hardlinks are not used at all or only for few files the fact that we store the
566 * table on disk shouldn't matter performance-wise. */
567
568 if (!FLAGS_SET(copy_flags, COPY_HARDLINKS))
569 return 0;
570
571 if (dt == AT_FDCWD)
572 dt_copy = AT_FDCWD;
573 else if (dt < 0)
574 return -EBADF;
575 else {
576 dt_copy = fcntl(dt, F_DUPFD_CLOEXEC, 3);
577 if (dt_copy < 0)
578 return -errno;
579 }
580
581 if (to)
582 r = tempfn_random_child(to, "hardlink", &c->subdir);
583 else
584 r = tempfn_random("hardlink", /* extra= */ NULL, &c->subdir);
585 if (r < 0)
586 return r;
587
588 c->parent_fd = TAKE_FD(dt_copy);
589
590 /* We don't actually create the directory we keep the table in here, that's done on-demand when the
591 * first entry is added, using hardlink_context_realize() below. */
592 return 1;
593 }
594
595 static int hardlink_context_realize(HardlinkContext *c) {
596 if (!c)
597 return 0;
598
599 if (c->dir_fd >= 0) /* Already realized */
600 return 1;
601
602 if (c->parent_fd < 0 && c->parent_fd != AT_FDCWD) /* Not configured */
603 return 0;
604
605 assert(c->subdir);
606
607 c->dir_fd = open_mkdir_at(c->parent_fd, c->subdir, O_EXCL|O_CLOEXEC, 0700);
608 if (c->dir_fd < 0)
609 return c->dir_fd;
610
611 return 1;
612 }
613
614 static void hardlink_context_destroy(HardlinkContext *c) {
615 int r;
616
617 assert(c);
618
619 /* Automatically remove the hardlink lookup table directory again after we are done. This is used via
620 * _cleanup_() so that we really delete this, even on failure. */
621
622 if (c->dir_fd >= 0) {
623 /* <dir_fd> might be have already been used for reading, so we need to rewind it. */
624 if (lseek(c->dir_fd, 0, SEEK_SET) < 0)
625 log_debug_errno(errno, "Failed to lseek on file descriptor, ignoring: %m");
626
627 r = rm_rf_children(TAKE_FD(c->dir_fd), REMOVE_PHYSICAL, NULL); /* consumes dir_fd in all cases, even on failure */
628 if (r < 0)
629 log_debug_errno(r, "Failed to remove hardlink store (%s) contents, ignoring: %m", c->subdir);
630
631 assert(c->parent_fd >= 0 || c->parent_fd == AT_FDCWD);
632 assert(c->subdir);
633
634 if (unlinkat(c->parent_fd, c->subdir, AT_REMOVEDIR) < 0)
635 log_debug_errno(errno, "Failed to remove hardlink store (%s) directory, ignoring: %m", c->subdir);
636 }
637
638 assert_cc(AT_FDCWD < 0);
639 c->parent_fd = safe_close(c->parent_fd);
640
641 c->subdir = mfree(c->subdir);
642 }
643
644 static int try_hardlink(
645 HardlinkContext *c,
646 const struct stat *st,
647 int dt,
648 const char *to) {
649
650 char dev_ino[DECIMAL_STR_MAX(dev_t)*2 + DECIMAL_STR_MAX(uint64_t) + 4];
651
652 assert(st);
653 assert(dt >= 0 || dt == AT_FDCWD);
654 assert(to);
655
656 if (!c) /* No temporary hardlink directory, don't bother */
657 return 0;
658
659 if (st->st_nlink <= 1) /* Source not hardlinked, don't bother */
660 return 0;
661
662 if (c->dir_fd < 0) /* not yet realized, hence empty */
663 return 0;
664
665 xsprintf(dev_ino, "%u:%u:%" PRIu64, major(st->st_dev), minor(st->st_dev), (uint64_t) st->st_ino);
666 if (linkat(c->dir_fd, dev_ino, dt, to, 0) < 0) {
667 if (errno != ENOENT) /* doesn't exist in store yet */
668 log_debug_errno(errno, "Failed to hardlink %s to %s, ignoring: %m", dev_ino, to);
669 return 0;
670 }
671
672 return 1;
673 }
674
675 static int memorize_hardlink(
676 HardlinkContext *c,
677 const struct stat *st,
678 int dt,
679 const char *to) {
680
681 char dev_ino[DECIMAL_STR_MAX(dev_t)*2 + DECIMAL_STR_MAX(uint64_t) + 4];
682 int r;
683
684 assert(st);
685 assert(dt >= 0 || dt == AT_FDCWD);
686 assert(to);
687
688 if (!c) /* No temporary hardlink directory, don't bother */
689 return 0;
690
691 if (st->st_nlink <= 1) /* Source not hardlinked, don't bother */
692 return 0;
693
694 r = hardlink_context_realize(c); /* Create the hardlink store lazily */
695 if (r < 0)
696 return r;
697
698 xsprintf(dev_ino, "%u:%u:%" PRIu64, major(st->st_dev), minor(st->st_dev), (uint64_t) st->st_ino);
699 if (linkat(dt, to, c->dir_fd, dev_ino, 0) < 0) {
700 log_debug_errno(errno, "Failed to hardlink %s to %s, ignoring: %m", to, dev_ino);
701 return 0;
702 }
703
704 return 1;
705 }
706
707 static int prepare_nocow(int fdf, const char *from, int fdt, unsigned *chattr_mask, unsigned *chattr_flags) {
708 unsigned attrs = 0;
709 int r;
710
711 assert(fdf >= 0 || fdf == AT_FDCWD);
712 assert(fdt >= 0);
713 assert(!!chattr_mask == !!chattr_flags);
714
715 /* If caller explicitly requested NOCOW to be set or unset, let's not interfere. */
716 if (chattr_mask && FLAGS_SET(*chattr_mask, FS_NOCOW_FL))
717 return 0;
718
719 r = read_attr_at(fdf, from, &attrs);
720 if (r < 0 && !ERRNO_IS_IOCTL_NOT_SUPPORTED(r) && r != -ELOOP) /* If the source is a symlink we get ELOOP */
721 return r;
722
723 if (FLAGS_SET(attrs, FS_NOCOW_FL)) {
724 if (chattr_mask && chattr_flags) {
725 *chattr_mask |= FS_NOCOW_FL;
726 *chattr_flags |= FS_NOCOW_FL;
727 } else
728 /* If the NOCOW flag is set on the source, make the copy NOCOW as well. If the source
729 * is not NOCOW, don't do anything in particular with the copy. */
730 (void) chattr_fd(fdt, FS_NOCOW_FL, FS_NOCOW_FL);
731 }
732
733 return 0;
734 }
735
736 /* Copies fs-verity status. May re-open fdt to do its job. */
737 static int copy_fs_verity(int fdf, int *fdt) {
738 int r;
739
740 assert(fdf >= 0);
741 assert(fdt);
742 assert(*fdt >= 0);
743
744 r = fd_verify_regular(fdf);
745 if (r < 0)
746 return r;
747
748 struct fsverity_descriptor desc = {};
749 struct fsverity_read_metadata_arg read_arg = {
750 .metadata_type = FS_VERITY_METADATA_TYPE_DESCRIPTOR,
751 .buf_ptr = (uintptr_t) &desc,
752 .length = sizeof(desc),
753 };
754
755 r = ioctl(fdf, FS_IOC_READ_VERITY_METADATA, &read_arg);
756 if (r < 0) {
757 /* ENODATA means that the file doesn't have fs-verity,
758 * so the correct thing to do is to do nothing at all. */
759 if (errno == ENODATA)
760 return 0;
761 log_error_errno(errno, "Failed to read fs-verity metadata from source file: %m");
762 /* For cases where fs-verity is unsupported we return a special error code */
763 if (ERRNO_IS_NOT_SUPPORTED(errno))
764 return -ESOCKTNOSUPPORT;
765 return -errno;
766 }
767
768 /* Make sure that the descriptor is completely initialized */
769 assert(r == (int) sizeof desc);
770
771 r = fd_verify_regular(*fdt);
772 if (r < 0)
773 return r;
774
775 /* Okay. We're doing this now. We need to re-open fdt as read-only because
776 * we can't enable fs-verity while writable file descriptors are outstanding. */
777 _cleanup_close_ int reopened_fd = -EBADF;
778 r = fd_reopen_condition(*fdt, O_RDONLY|O_CLOEXEC|O_NOCTTY, O_ACCMODE_STRICT|O_PATH, &reopened_fd);
779 if (r < 0)
780 return r;
781 if (reopened_fd >= 0)
782 close_and_replace(*fdt, reopened_fd);
783
784 struct fsverity_enable_arg enable_arg = {
785 .version = desc.version,
786 .hash_algorithm = desc.hash_algorithm,
787 .block_size = UINT32_C(1) << desc.log_blocksize,
788 .salt_size = desc.salt_size,
789 .salt_ptr = (uintptr_t) &desc.salt,
790 };
791
792 if (ioctl(*fdt, FS_IOC_ENABLE_VERITY, &enable_arg) < 0) {
793 log_error_errno(errno, "Failed to set fs-verity metadata: %m");
794 /* For cases where fs-verity is unsupported we return a special error code */
795 if (ERRNO_IS_NOT_SUPPORTED(errno))
796 return -ESOCKTNOSUPPORT;
797 return -errno;
798 }
799
800 return 0;
801 }
802
803 static int fd_copy_tree_generic(
804 int df,
805 const char *from,
806 const struct stat *st,
807 int dt,
808 const char *to,
809 dev_t original_device,
810 unsigned depth_left,
811 uid_t override_uid,
812 gid_t override_gid,
813 CopyFlags copy_flags,
814 Hashmap *denylist,
815 Hashmap *subvolumes,
816 HardlinkContext *hardlink_context,
817 const char *display_path,
818 copy_progress_path_t progress_path,
819 copy_progress_bytes_t progress_bytes,
820 void *userdata);
821
822 static int fd_copy_regular(
823 int df,
824 const char *from,
825 const struct stat *st,
826 int dt,
827 const char *to,
828 uid_t override_uid,
829 gid_t override_gid,
830 CopyFlags copy_flags,
831 HardlinkContext *hardlink_context,
832 copy_progress_bytes_t progress,
833 void *userdata) {
834
835 _cleanup_close_ int fdf = -EBADF, fdt = -EBADF;
836 int r, q;
837
838 assert(st);
839 assert(to);
840
841 r = try_hardlink(hardlink_context, st, dt, to);
842 if (r < 0)
843 return r;
844 if (r > 0) /* worked! */
845 return 0;
846
847 fdf = xopenat_full(df, from, O_RDONLY|O_CLOEXEC|O_NOCTTY|O_NOFOLLOW, XO_REGULAR, 0);
848 if (fdf < 0)
849 return fdf;
850
851 if (copy_flags & COPY_MAC_CREATE) {
852 r = mac_selinux_create_file_prepare_at(dt, to, S_IFREG);
853 if (r < 0)
854 return r;
855 }
856 fdt = openat(dt, to, O_WRONLY|O_CREAT|O_EXCL|O_CLOEXEC|O_NOCTTY|O_NOFOLLOW, st->st_mode & 07777);
857 if (copy_flags & COPY_MAC_CREATE)
858 mac_selinux_create_file_clear();
859 if (fdt < 0)
860 return -errno;
861
862 r = prepare_nocow(fdf, /* from= */ NULL, fdt, /* chattr_mask= */ NULL, /* chattr_flags= */ NULL);
863 if (r < 0)
864 return r;
865
866 r = copy_bytes_full(fdf, fdt, UINT64_MAX, copy_flags, NULL, NULL, progress, userdata);
867 if (r < 0)
868 goto fail;
869
870 if (fchown(fdt,
871 uid_is_valid(override_uid) ? override_uid : st->st_uid,
872 gid_is_valid(override_gid) ? override_gid : st->st_gid) < 0)
873 r = -errno;
874
875 if (fchmod(fdt, st->st_mode & 07777) < 0)
876 r = -errno;
877
878 (void) futimens(fdt, (struct timespec[]) { st->st_atim, st->st_mtim });
879 (void) copy_xattr(fdf, NULL, fdt, NULL, copy_flags);
880
881 if (FLAGS_SET(copy_flags, COPY_VERIFY_LINKED)) {
882 r = fd_verify_linked(fdf);
883 if (r < 0)
884 return r;
885 }
886
887 /* NB: fs-verity cannot be enabled when a writable file descriptor is outstanding.
888 * copy_fs_verity() may well re-open 'fdt' as O_RDONLY. All code below this point
889 * needs to be able to work with a read-only file descriptor. */
890 if (FLAGS_SET(copy_flags, COPY_PRESERVE_FS_VERITY)) {
891 r = copy_fs_verity(fdf, &fdt);
892 if (r < 0)
893 goto fail;
894 }
895
896 if (copy_flags & COPY_FSYNC) {
897 if (fsync(fdt) < 0) {
898 r = -errno;
899 goto fail;
900 }
901 }
902
903 q = close_nointr(TAKE_FD(fdt)); /* even if this fails, the fd is now invalidated */
904 if (q < 0) {
905 r = q;
906 goto fail;
907 }
908
909 (void) memorize_hardlink(hardlink_context, st, dt, to);
910 return r;
911
912 fail:
913 (void) unlinkat(dt, to, 0);
914 return r;
915 }
916
917 static int fd_copy_fifo(
918 int df,
919 const char *from,
920 const struct stat *st,
921 int dt,
922 const char *to,
923 uid_t override_uid,
924 gid_t override_gid,
925 CopyFlags copy_flags,
926 HardlinkContext *hardlink_context) {
927 int r;
928
929 assert(st);
930 assert(to);
931
932 r = try_hardlink(hardlink_context, st, dt, to);
933 if (r < 0)
934 return r;
935 if (r > 0) /* worked! */
936 return 0;
937
938 if (copy_flags & COPY_MAC_CREATE) {
939 r = mac_selinux_create_file_prepare_at(dt, to, S_IFIFO);
940 if (r < 0)
941 return r;
942 }
943 r = RET_NERRNO(mkfifoat(dt, to, st->st_mode & 07777));
944 if (copy_flags & COPY_MAC_CREATE)
945 mac_selinux_create_file_clear();
946 if (FLAGS_SET(copy_flags, COPY_GRACEFUL_WARN) && (ERRNO_IS_NEG_PRIVILEGE(r) || ERRNO_IS_NEG_NOT_SUPPORTED(r))) {
947 log_notice_errno(r, "Failed to copy fifo%s%s%s, ignoring: %m",
948 isempty(from) ? "" : " '",
949 strempty(from),
950 isempty(from) ? "" : "'");
951 return 0;
952 } else if (r < 0)
953 return r;
954
955 if (fchownat(dt, to,
956 uid_is_valid(override_uid) ? override_uid : st->st_uid,
957 gid_is_valid(override_gid) ? override_gid : st->st_gid,
958 AT_SYMLINK_NOFOLLOW) < 0)
959 r = -errno;
960
961 if (fchmodat(dt, to, st->st_mode & 07777, 0) < 0)
962 r = -errno;
963
964 (void) utimensat(dt, to, (struct timespec[]) { st->st_atim, st->st_mtim }, AT_SYMLINK_NOFOLLOW);
965
966 (void) memorize_hardlink(hardlink_context, st, dt, to);
967 return r;
968 }
969
970 static int fd_copy_node(
971 int df,
972 const char *from,
973 const struct stat *st,
974 int dt,
975 const char *to,
976 uid_t override_uid,
977 gid_t override_gid,
978 CopyFlags copy_flags,
979 HardlinkContext *hardlink_context) {
980 int r;
981
982 assert(st);
983 assert(to);
984
985 r = try_hardlink(hardlink_context, st, dt, to);
986 if (r < 0)
987 return r;
988 if (r > 0) /* worked! */
989 return 0;
990
991 if (copy_flags & COPY_MAC_CREATE) {
992 r = mac_selinux_create_file_prepare_at(dt, to, st->st_mode & S_IFMT);
993 if (r < 0)
994 return r;
995 }
996 r = RET_NERRNO(mknodat(dt, to, st->st_mode, st->st_rdev));
997 if (copy_flags & COPY_MAC_CREATE)
998 mac_selinux_create_file_clear();
999 if (FLAGS_SET(copy_flags, COPY_GRACEFUL_WARN) && (ERRNO_IS_NEG_PRIVILEGE(r) || ERRNO_IS_NEG_NOT_SUPPORTED(r))) {
1000 log_notice_errno(r, "Failed to copy node%s%s%s, ignoring: %m",
1001 isempty(from) ? "" : " '",
1002 strempty(from),
1003 isempty(from) ? "" : "'");
1004 return 0;
1005 } else if (r < 0)
1006 return r;
1007
1008 if (fchownat(dt, to,
1009 uid_is_valid(override_uid) ? override_uid : st->st_uid,
1010 gid_is_valid(override_gid) ? override_gid : st->st_gid,
1011 AT_SYMLINK_NOFOLLOW) < 0)
1012 r = -errno;
1013
1014 if (fchmodat(dt, to, st->st_mode & 07777, 0) < 0)
1015 r = -errno;
1016
1017 (void) utimensat(dt, to, (struct timespec[]) { st->st_atim, st->st_mtim }, AT_SYMLINK_NOFOLLOW);
1018
1019 (void) memorize_hardlink(hardlink_context, st, dt, to);
1020 return r;
1021 }
1022
1023 static int fd_copy_directory(
1024 int df,
1025 const char *from,
1026 const struct stat *st,
1027 int dt,
1028 const char *to,
1029 dev_t original_device,
1030 unsigned depth_left,
1031 uid_t override_uid,
1032 gid_t override_gid,
1033 CopyFlags copy_flags,
1034 Hashmap *denylist,
1035 Hashmap *subvolumes,
1036 HardlinkContext *hardlink_context,
1037 const char *display_path,
1038 copy_progress_path_t progress_path,
1039 copy_progress_bytes_t progress_bytes,
1040 void *userdata) {
1041
1042 _cleanup_(hardlink_context_destroy) HardlinkContext our_hardlink_context = {
1043 .dir_fd = -EBADF,
1044 .parent_fd = -EBADF,
1045 };
1046
1047 _cleanup_close_ int fdf = -EBADF, fdt = -EBADF;
1048 _cleanup_closedir_ DIR *d = NULL;
1049 struct stat dt_st;
1050 bool exists;
1051 int r;
1052
1053 assert(st);
1054
1055 if (depth_left == 0)
1056 return -ENAMETOOLONG;
1057
1058 fdf = xopenat(df, from, O_RDONLY|O_DIRECTORY|O_CLOEXEC|O_NOCTTY|O_NOFOLLOW);
1059 if (fdf < 0)
1060 return fdf;
1061
1062 if (!hardlink_context) {
1063 /* If recreating hardlinks is requested let's set up a context for that now. */
1064 r = hardlink_context_setup(&our_hardlink_context, dt, to, copy_flags);
1065 if (r < 0)
1066 return r;
1067 if (r > 0) /* It's enabled and allocated, let's now use the same context for all recursive
1068 * invocations from here down */
1069 hardlink_context = &our_hardlink_context;
1070 }
1071
1072 d = take_fdopendir(&fdf);
1073 if (!d)
1074 return -errno;
1075
1076 r = dir_is_empty_at(dt, to, /* ignore_hidden_or_backup= */ false);
1077 if (r < 0 && r != -ENOENT)
1078 return r;
1079 if ((r > 0 && !(copy_flags & (COPY_MERGE|COPY_MERGE_EMPTY))) || (r == 0 && !FLAGS_SET(copy_flags, COPY_MERGE)))
1080 return -EEXIST;
1081
1082 exists = r >= 0;
1083
1084 XOpenFlags flags = copy_flags & COPY_MAC_CREATE ? XO_LABEL : 0;
1085 if (hashmap_contains(subvolumes, st)) {
1086 flags |= XO_SUBVOLUME;
1087 if ((PTR_TO_INT(hashmap_get(subvolumes, st)) & BTRFS_SUBVOL_NODATACOW))
1088 flags |= XO_NOCOW;
1089 }
1090
1091 fdt = xopenat_lock_full(dt, to,
1092 O_RDONLY|O_DIRECTORY|O_CLOEXEC|O_NOCTTY|O_NOFOLLOW|(exists ? 0 : O_CREAT|O_EXCL),
1093 flags,
1094 st->st_mode & 07777,
1095 copy_flags & COPY_LOCK_BSD ? LOCK_BSD : LOCK_NONE,
1096 LOCK_EX);
1097 if (fdt < 0)
1098 return fdt;
1099
1100 if (exists && FLAGS_SET(copy_flags, COPY_RESTORE_DIRECTORY_TIMESTAMPS) && fstat(fdt, &dt_st) < 0)
1101 return -errno;
1102
1103 int ret = 0;
1104
1105 if (PTR_TO_INT(hashmap_get(denylist, st)) == DENY_CONTENTS) {
1106 log_debug("%s is in the denylist, not recursing", from ?: "file to copy");
1107 goto finish;
1108 }
1109
1110 FOREACH_DIRENT_ALL(de, d, return -errno) {
1111 const char *child_display_path = NULL;
1112 _cleanup_free_ char *dp = NULL;
1113 struct stat buf;
1114
1115 if (dot_or_dot_dot(de->d_name))
1116 continue;
1117
1118 r = look_for_signals(copy_flags);
1119 if (r < 0)
1120 return r;
1121
1122 if (fstatat(dirfd(d), de->d_name, &buf, AT_SYMLINK_NOFOLLOW) < 0) {
1123 RET_GATHER(ret, -errno);
1124 continue;
1125 }
1126
1127 if (progress_path) {
1128 if (display_path)
1129 child_display_path = dp = path_join(display_path, de->d_name);
1130 else
1131 child_display_path = de->d_name;
1132
1133 r = progress_path(child_display_path, &buf, userdata);
1134 if (r < 0)
1135 return r;
1136 }
1137
1138 if (PTR_TO_INT(hashmap_get(denylist, &buf)) == DENY_INODE) {
1139 log_debug("%s%s%s is in the denylist, ignoring",
1140 strempty(from), isempty(from) ? "" : "/", de->d_name);
1141 continue;
1142 }
1143
1144 if (S_ISDIR(buf.st_mode)) {
1145 /*
1146 * Don't descend into directories on other file systems, if this is requested. We do a simple
1147 * .st_dev check here, which basically comes for free. Note that we do this check only on
1148 * directories, not other kind of file system objects, for two reason:
1149 *
1150 * • The kernel's overlayfs pseudo file system that overlays multiple real file systems
1151 * propagates the .st_dev field of the file system a file originates from all the way up
1152 * through the stack to stat(). It doesn't do that for directories however. This means that
1153 * comparing .st_dev on non-directories suggests that they all are mount points. To avoid
1154 * confusion we hence avoid relying on this check for regular files.
1155 *
1156 * • The main reason we do this check at all is to protect ourselves from bind mount cycles,
1157 * where we really want to avoid descending down in all eternity. However the .st_dev check
1158 * is usually not sufficient for this protection anyway, as bind mount cycles from the same
1159 * file system onto itself can't be detected that way. (Note we also do a recursion depth
1160 * check, which is probably the better protection in this regard, which is why
1161 * COPY_SAME_MOUNT is optional).
1162 */
1163
1164 if (FLAGS_SET(copy_flags, COPY_SAME_MOUNT)) {
1165 if (buf.st_dev != original_device)
1166 continue;
1167
1168 r = is_mount_point_at(dirfd(d), de->d_name, 0);
1169 if (r < 0)
1170 return r;
1171 if (r > 0)
1172 continue;
1173 }
1174 }
1175
1176 r = fd_copy_tree_generic(dirfd(d), de->d_name, &buf, fdt, de->d_name, original_device,
1177 depth_left-1, override_uid, override_gid, copy_flags & ~COPY_LOCK_BSD,
1178 denylist, subvolumes, hardlink_context, child_display_path, progress_path,
1179 progress_bytes, userdata);
1180
1181 /* Propagate SIGINT/SIGTERM, ENOSPC, and fs-verity fails up instantly */
1182 if (IN_SET(r, -EINTR, -ENOSPC, -ESOCKTNOSUPPORT))
1183 return r;
1184 if (r == -EEXIST && (copy_flags & COPY_MERGE))
1185 r = 0;
1186 RET_GATHER(ret, r);
1187 }
1188
1189 finish:
1190 if (FLAGS_SET(copy_flags, COPY_MERGE_APPLY_STAT) || !exists) {
1191 if (fchown(fdt,
1192 uid_is_valid(override_uid) ? override_uid : st->st_uid,
1193 gid_is_valid(override_gid) ? override_gid : st->st_gid) < 0)
1194 RET_GATHER(ret, -errno);
1195
1196 if (fchmod(fdt, st->st_mode & 07777) < 0)
1197 RET_GATHER(ret, -errno);
1198
1199 /* Run hardlink context cleanup now because it potentially changes timestamps */
1200 hardlink_context_destroy(&our_hardlink_context);
1201 (void) copy_xattr(dirfd(d), NULL, fdt, NULL, copy_flags);
1202 (void) futimens(fdt, (struct timespec[]) { st->st_atim, st->st_mtim });
1203 } else if (FLAGS_SET(copy_flags, COPY_RESTORE_DIRECTORY_TIMESTAMPS)) {
1204 /* Run hardlink context cleanup now because it potentially changes timestamps */
1205 hardlink_context_destroy(&our_hardlink_context);
1206 /* If the directory already exists, make sure the timestamps stay the same as before. */
1207 (void) futimens(fdt, (struct timespec[]) { dt_st.st_atim, dt_st.st_mtim });
1208 }
1209
1210 if (copy_flags & COPY_FSYNC_FULL) {
1211 if (fsync(fdt) < 0)
1212 return -errno;
1213 }
1214
1215 if (ret < 0)
1216 return ret;
1217
1218 return copy_flags & COPY_LOCK_BSD ? TAKE_FD(fdt) : 0;
1219 }
1220
1221 static int fd_copy_leaf(
1222 int df,
1223 const char *from,
1224 const struct stat *st,
1225 int dt,
1226 const char *to,
1227 uid_t override_uid,
1228 gid_t override_gid,
1229 CopyFlags copy_flags,
1230 HardlinkContext *hardlink_context,
1231 const char *display_path,
1232 copy_progress_bytes_t progress_bytes,
1233 void *userdata) {
1234 int r;
1235
1236 if (S_ISREG(st->st_mode))
1237 r = fd_copy_regular(df, from, st, dt, to, override_uid, override_gid, copy_flags, hardlink_context, progress_bytes, userdata);
1238 else if (S_ISLNK(st->st_mode))
1239 r = fd_copy_symlink(df, from, st, dt, to, override_uid, override_gid, copy_flags);
1240 else if (S_ISFIFO(st->st_mode))
1241 r = fd_copy_fifo(df, from, st, dt, to, override_uid, override_gid, copy_flags, hardlink_context);
1242 else if (S_ISBLK(st->st_mode) || S_ISCHR(st->st_mode) || S_ISSOCK(st->st_mode))
1243 r = fd_copy_node(df, from, st, dt, to, override_uid, override_gid, copy_flags, hardlink_context);
1244 else
1245 r = -EOPNOTSUPP;
1246
1247 return r;
1248 }
1249
1250 static int fd_copy_tree_generic(
1251 int df,
1252 const char *from,
1253 const struct stat *st,
1254 int dt,
1255 const char *to,
1256 dev_t original_device,
1257 unsigned depth_left,
1258 uid_t override_uid,
1259 gid_t override_gid,
1260 CopyFlags copy_flags,
1261 Hashmap *denylist,
1262 Hashmap *subvolumes,
1263 HardlinkContext *hardlink_context,
1264 const char *display_path,
1265 copy_progress_path_t progress_path,
1266 copy_progress_bytes_t progress_bytes,
1267 void *userdata) {
1268
1269 int r;
1270
1271 assert(!FLAGS_SET(copy_flags, COPY_LOCK_BSD));
1272
1273 if (S_ISDIR(st->st_mode))
1274 return fd_copy_directory(df, from, st, dt, to, original_device, depth_left-1, override_uid,
1275 override_gid, copy_flags, denylist, subvolumes, hardlink_context,
1276 display_path, progress_path, progress_bytes, userdata);
1277
1278 /* Only if we are copying a directory we are fine if the target dir is referenced by fd only */
1279 if (!to)
1280 return -ENOTDIR;
1281
1282 DenyType t = PTR_TO_INT(hashmap_get(denylist, st));
1283 if (t == DENY_INODE) {
1284 log_debug("%s is in the denylist, ignoring", from ?: "file to copy");
1285 return 0;
1286 } else if (t == DENY_CONTENTS)
1287 log_debug("%s is configured to have its contents excluded, but is not a directory", from ?: "file to copy");
1288
1289 r = fd_copy_leaf(df, from, st, dt, to, override_uid, override_gid, copy_flags, hardlink_context, display_path, progress_bytes, userdata);
1290 /* We just tried to copy a leaf node of the tree. If it failed because the node already exists *and* the COPY_REPLACE flag has been provided, we should unlink the node and re-copy. */
1291 if (r == -EEXIST && (copy_flags & COPY_REPLACE)) {
1292 /* This codepath is us trying to address an error to copy, if the unlink fails, lets just return the original error. */
1293 if (unlinkat(dt, to, 0) < 0)
1294 return r;
1295
1296 r = fd_copy_leaf(df, from, st, dt, to, override_uid, override_gid, copy_flags, hardlink_context, display_path, progress_bytes, userdata);
1297 }
1298
1299 return r;
1300 }
1301
1302 int copy_tree_at_full(
1303 int fdf,
1304 const char *from,
1305 int fdt,
1306 const char *to,
1307 uid_t override_uid,
1308 gid_t override_gid,
1309 CopyFlags copy_flags,
1310 Hashmap *denylist,
1311 Hashmap *subvolumes,
1312 copy_progress_path_t progress_path,
1313 copy_progress_bytes_t progress_bytes,
1314 void *userdata) {
1315
1316 struct stat st;
1317 int r;
1318
1319 assert(!FLAGS_SET(copy_flags, COPY_LOCK_BSD));
1320
1321 if (fstatat(fdf, strempty(from), &st, AT_SYMLINK_NOFOLLOW | (isempty(from) ? AT_EMPTY_PATH : 0)) < 0)
1322 return -errno;
1323
1324 r = fd_copy_tree_generic(fdf, from, &st, fdt, to, st.st_dev, COPY_DEPTH_MAX, override_uid,
1325 override_gid, copy_flags, denylist, subvolumes, NULL, NULL, progress_path,
1326 progress_bytes, userdata);
1327 if (r < 0)
1328 return r;
1329
1330 if (S_ISDIR(st.st_mode) && (copy_flags & COPY_SYNCFS)) {
1331 /* If the top-level inode is a directory run syncfs() now. */
1332 r = syncfs_path(fdt, to);
1333 if (r < 0)
1334 return r;
1335 } else if ((copy_flags & (COPY_FSYNC_FULL|COPY_SYNCFS)) != 0) {
1336 /* fsync() the parent dir of what we just copied if COPY_FSYNC_FULL is set. Also do this in
1337 * case COPY_SYNCFS is set but the top-level inode wasn't actually a directory. We do this so that
1338 * COPY_SYNCFS provides reasonable synchronization semantics on any kind of inode: when the
1339 * copy operation is done the whole inode — regardless of its type — and all its children
1340 * will be synchronized to disk. */
1341 r = fsync_parent_at(fdt, to);
1342 if (r < 0)
1343 return r;
1344 }
1345
1346 return 0;
1347 }
1348
1349 static int sync_dir_by_flags(int dir_fd, const char *path, CopyFlags copy_flags) {
1350 assert(dir_fd >= 0 || dir_fd == AT_FDCWD);
1351 assert(path);
1352
1353 if (copy_flags & COPY_SYNCFS)
1354 return syncfs_path(dir_fd, path);
1355 if (copy_flags & COPY_FSYNC_FULL)
1356 return fsync_parent_at(dir_fd, path);
1357
1358 return 0;
1359 }
1360
1361 int copy_directory_at_full(
1362 int dir_fdf,
1363 const char *from,
1364 int dir_fdt,
1365 const char *to,
1366 uid_t override_uid,
1367 gid_t override_gid,
1368 CopyFlags copy_flags,
1369 copy_progress_path_t progress_path,
1370 copy_progress_bytes_t progress_bytes,
1371 void *userdata) {
1372
1373 _cleanup_close_ int fdt = -EBADF;
1374 struct stat st;
1375 int r;
1376
1377 assert(dir_fdf >= 0 || dir_fdf == AT_FDCWD);
1378 assert(dir_fdt >= 0 || dir_fdt == AT_FDCWD);
1379 assert(to);
1380
1381 if (fstatat(dir_fdf, strempty(from), &st, AT_SYMLINK_NOFOLLOW|(isempty(from) ? AT_EMPTY_PATH : 0)) < 0)
1382 return -errno;
1383
1384 r = stat_verify_directory(&st);
1385 if (r < 0)
1386 return r;
1387
1388 r = fd_copy_directory(
1389 dir_fdf, from,
1390 &st,
1391 dir_fdt, to,
1392 st.st_dev,
1393 COPY_DEPTH_MAX,
1394 override_uid,
1395 override_gid,
1396 copy_flags,
1397 /* denylist= */ NULL,
1398 /* subvolumes= */ NULL,
1399 /* progress_path= */ NULL,
1400 /* progress_bytes= */ NULL,
1401 progress_path,
1402 progress_bytes,
1403 userdata);
1404 if (r < 0)
1405 return r;
1406
1407 if (FLAGS_SET(copy_flags, COPY_LOCK_BSD))
1408 fdt = r;
1409
1410 r = sync_dir_by_flags(dir_fdt, to, copy_flags);
1411 if (r < 0)
1412 return r;
1413
1414 return FLAGS_SET(copy_flags, COPY_LOCK_BSD) ? TAKE_FD(fdt) : 0;
1415 }
1416
1417 int copy_file_fd_at_full(
1418 int dir_fdf,
1419 const char *from,
1420 int fdt,
1421 CopyFlags copy_flags,
1422 copy_progress_bytes_t progress_bytes,
1423 void *userdata) {
1424
1425 _cleanup_close_ int fdf = -EBADF;
1426 struct stat st;
1427 int r;
1428
1429 assert(dir_fdf >= 0 || dir_fdf == AT_FDCWD);
1430 assert(fdt >= 0);
1431 assert(!FLAGS_SET(copy_flags, COPY_LOCK_BSD));
1432
1433 fdf = xopenat_full(dir_fdf, from, O_RDONLY|O_CLOEXEC|O_NOCTTY, XO_REGULAR, 0);
1434 if (fdf < 0)
1435 return fdf;
1436
1437 if (fstat(fdt, &st) < 0)
1438 return -errno;
1439
1440 r = copy_bytes_full(fdf, fdt, UINT64_MAX, copy_flags, NULL, NULL, progress_bytes, userdata);
1441 if (r < 0)
1442 return r;
1443
1444 /* Make sure to copy file attributes only over if target is a regular
1445 * file (so that copying a file to /dev/null won't alter the access
1446 * mode/ownership of that device node...) */
1447 if (S_ISREG(st.st_mode)) {
1448 (void) copy_times(fdf, fdt, copy_flags);
1449 (void) copy_xattr(fdf, NULL, fdt, NULL, copy_flags);
1450 }
1451
1452 if (FLAGS_SET(copy_flags, COPY_VERIFY_LINKED)) {
1453 r = fd_verify_linked(fdf);
1454 if (r < 0)
1455 return r;
1456 }
1457
1458 if (copy_flags & COPY_FSYNC_FULL) {
1459 r = fsync_full(fdt);
1460 if (r < 0)
1461 return r;
1462 } else if (copy_flags & COPY_FSYNC) {
1463 if (fsync(fdt) < 0)
1464 return -errno;
1465 }
1466
1467 return 0;
1468 }
1469
1470 int copy_file_at_full(
1471 int dir_fdf,
1472 const char *from,
1473 int dir_fdt,
1474 const char *to,
1475 int flags,
1476 mode_t mode,
1477 unsigned chattr_flags,
1478 unsigned chattr_mask,
1479 CopyFlags copy_flags,
1480 copy_progress_bytes_t progress_bytes,
1481 void *userdata) {
1482
1483 _cleanup_close_ int fdf = -EBADF, fdt = -EBADF;
1484 int r;
1485
1486 assert(dir_fdf >= 0 || dir_fdf == AT_FDCWD);
1487 assert(dir_fdt >= 0 || dir_fdt == AT_FDCWD);
1488 assert(to);
1489
1490 fdf = xopenat_full(dir_fdf, from, O_RDONLY|O_CLOEXEC|O_NOCTTY, XO_REGULAR, 0);
1491 if (fdf < 0)
1492 return fdf;
1493
1494 if (mode == MODE_INVALID) {
1495 struct stat st;
1496
1497 if (fstat(fdf, &st) < 0)
1498 return -errno;
1499
1500 mode = st.st_mode;
1501 }
1502
1503 WITH_UMASK(0000) {
1504 fdt = xopenat_lock_full(dir_fdt, to,
1505 flags|O_WRONLY|O_CREAT|O_CLOEXEC|O_NOCTTY,
1506 XO_REGULAR | (copy_flags & COPY_MAC_CREATE ? XO_LABEL : 0),
1507 mode,
1508 copy_flags & COPY_LOCK_BSD ? LOCK_BSD : LOCK_NONE, LOCK_EX);
1509 if (fdt < 0)
1510 return fdt;
1511 }
1512
1513 r = prepare_nocow(fdf, /* from= */ NULL, fdt, &chattr_mask, &chattr_flags);
1514 if (r < 0)
1515 return r;
1516
1517 if ((chattr_mask & CHATTR_EARLY_FL) != 0)
1518 (void) chattr_fd(fdt, chattr_flags, chattr_mask & CHATTR_EARLY_FL);
1519
1520 r = copy_bytes_full(fdf, fdt, UINT64_MAX, copy_flags & ~COPY_LOCK_BSD, NULL, NULL, progress_bytes, userdata);
1521 if (r < 0)
1522 goto fail;
1523
1524 (void) copy_times(fdf, fdt, copy_flags);
1525 (void) copy_xattr(fdf, NULL, fdt, NULL, copy_flags);
1526
1527 if (FLAGS_SET(copy_flags, COPY_VERIFY_LINKED)) {
1528 r = fd_verify_linked(fdf);
1529 if (r < 0)
1530 goto fail;
1531 }
1532
1533 unsigned nocow = FLAGS_SET(copy_flags, COPY_NOCOW_AFTER) ? FS_NOCOW_FL : 0;
1534 if (((chattr_mask & ~CHATTR_EARLY_FL) | nocow) != 0)
1535 (void) chattr_fd(fdt, chattr_flags | nocow, (chattr_mask & ~CHATTR_EARLY_FL) | nocow);
1536
1537 if (copy_flags & (COPY_FSYNC|COPY_FSYNC_FULL)) {
1538 if (fsync(fdt) < 0) {
1539 r = -errno;
1540 goto fail;
1541 }
1542 }
1543
1544 if (!FLAGS_SET(copy_flags, COPY_LOCK_BSD)) {
1545 r = close_nointr(TAKE_FD(fdt)); /* even if this fails, the fd is now invalidated */
1546 if (r < 0)
1547 goto fail;
1548 }
1549
1550 if (copy_flags & COPY_FSYNC_FULL) {
1551 r = fsync_parent_at(dir_fdt, to);
1552 if (r < 0)
1553 goto fail;
1554 }
1555
1556 return copy_flags & COPY_LOCK_BSD ? TAKE_FD(fdt) : 0;
1557
1558 fail:
1559 /* Only unlink if we definitely are the ones who created the file */
1560 if (FLAGS_SET(flags, O_EXCL))
1561 (void) unlinkat(dir_fdt, to, 0);
1562
1563 return r;
1564 }
1565
1566 int copy_file_atomic_at_full(
1567 int dir_fdf,
1568 const char *from,
1569 int dir_fdt,
1570 const char *to,
1571 mode_t mode,
1572 unsigned chattr_flags,
1573 unsigned chattr_mask,
1574 CopyFlags copy_flags,
1575 copy_progress_bytes_t progress_bytes,
1576 void *userdata) {
1577
1578 _cleanup_(unlink_and_freep) char *t = NULL;
1579 _cleanup_close_ int fdt = -EBADF;
1580 int r;
1581
1582 assert(to);
1583 assert(!FLAGS_SET(copy_flags, COPY_LOCK_BSD));
1584
1585 if (copy_flags & COPY_MAC_CREATE) {
1586 r = mac_selinux_create_file_prepare_at(dir_fdt, to, S_IFREG);
1587 if (r < 0)
1588 return r;
1589 }
1590 fdt = open_tmpfile_linkable_at(dir_fdt, to, O_WRONLY|O_CLOEXEC, &t);
1591 if (copy_flags & COPY_MAC_CREATE)
1592 mac_selinux_create_file_clear();
1593 if (fdt < 0)
1594 return fdt;
1595
1596 r = prepare_nocow(dir_fdf, from, fdt, &chattr_mask, &chattr_flags);
1597 if (r < 0)
1598 return r;
1599
1600 if ((chattr_mask & CHATTR_EARLY_FL) != 0)
1601 (void) chattr_fd(fdt, chattr_flags, chattr_mask & CHATTR_EARLY_FL);
1602
1603 r = copy_file_fd_at_full(dir_fdf, from, fdt, copy_flags, progress_bytes, userdata);
1604 if (r < 0)
1605 return r;
1606
1607 if (fchmod(fdt, mode) < 0)
1608 return -errno;
1609
1610 if ((copy_flags & (COPY_FSYNC|COPY_FSYNC_FULL))) {
1611 /* Sync the file */
1612 if (fsync(fdt) < 0)
1613 return -errno;
1614 }
1615
1616 r = link_tmpfile_at(fdt, dir_fdt, t, to, (copy_flags & COPY_REPLACE) ? LINK_TMPFILE_REPLACE : 0);
1617 if (r < 0)
1618 return r;
1619
1620 t = mfree(t);
1621
1622 unsigned nocow = FLAGS_SET(copy_flags, COPY_NOCOW_AFTER) ? FS_NOCOW_FL : 0;
1623 if (((chattr_mask & ~CHATTR_EARLY_FL) | nocow) != 0)
1624 (void) chattr_fd(fdt, chattr_flags | nocow, (chattr_mask & ~CHATTR_EARLY_FL) | nocow);
1625
1626 r = close_nointr(TAKE_FD(fdt)); /* even if this fails, the fd is now invalidated */
1627 if (r < 0)
1628 goto fail;
1629
1630 if (copy_flags & COPY_FSYNC_FULL) {
1631 /* Sync the parent directory */
1632 r = fsync_parent_at(dir_fdt, to);
1633 if (r < 0)
1634 goto fail;
1635 }
1636
1637 return 0;
1638
1639 fail:
1640 (void) unlinkat(dir_fdt, to, 0);
1641 return r;
1642 }
1643
1644 int copy_times(int fdf, int fdt, CopyFlags flags) {
1645 struct stat st;
1646
1647 assert(fdf >= 0);
1648 assert(fdt >= 0);
1649
1650 if (fstat(fdf, &st) < 0)
1651 return -errno;
1652
1653 if (futimens(fdt, (struct timespec[2]) { st.st_atim, st.st_mtim }) < 0)
1654 return -errno;
1655
1656 if (FLAGS_SET(flags, COPY_CRTIME)) {
1657 usec_t crtime;
1658
1659 if (fd_getcrtime(fdf, &crtime) >= 0)
1660 (void) fd_setcrtime(fdt, crtime);
1661 }
1662
1663 return 0;
1664 }
1665
1666 int copy_access(int fdf, int fdt) {
1667 struct stat st;
1668
1669 assert(fdf >= 0);
1670 assert(fdt >= 0);
1671
1672 /* Copies just the access mode (and not the ownership) from fdf to fdt */
1673
1674 if (fstat(fdf, &st) < 0)
1675 return -errno;
1676
1677 return RET_NERRNO(fchmod(fdt, st.st_mode & 07777));
1678 }
1679
1680 int copy_owner(int fdf, int fdt) {
1681 struct stat st;
1682
1683 assert(fdf >= 0);
1684 assert(fdt >= 0);
1685
1686 if (fstat(fdf, &st) < 0)
1687 return -errno;
1688
1689 return RET_NERRNO(fchown(fdt, st.st_uid, st.st_gid));
1690 }
1691
1692 int copy_rights_with_fallback(int fdf, int fdt, const char *patht) {
1693 struct stat st;
1694
1695 assert(fdf >= 0);
1696 assert(fdt >= 0);
1697
1698 /* Copies both access mode and ownership from fdf to fdt */
1699
1700 if (fstat(fdf, &st) < 0)
1701 return -errno;
1702
1703 return fchmod_and_chown_with_fallback(fdt, patht, st.st_mode & 07777, st.st_uid, st.st_gid);
1704 }
1705
1706 int copy_xattr(int df, const char *from, int dt, const char *to, CopyFlags copy_flags) {
1707 _cleanup_free_ char *names = NULL;
1708 int ret = 0, r;
1709
1710 r = listxattr_at_malloc(df, from, 0, &names);
1711 if (r < 0)
1712 return r;
1713
1714 NULSTR_FOREACH(p, names) {
1715 if (!FLAGS_SET(copy_flags, COPY_ALL_XATTRS) && !startswith(p, "user."))
1716 continue;
1717
1718 _cleanup_free_ char *value = NULL;
1719 size_t value_size;
1720 r = getxattr_at_malloc(df, from, p, 0, &value, &value_size);
1721 if (r == -ENODATA)
1722 continue; /* gone by now */
1723 if (r < 0)
1724 return r;
1725
1726 RET_GATHER(ret, xsetxattr_full(dt, to, /* at_flags= */ 0, p, value, value_size, /* xattr_flags= */ 0));
1727 }
1728
1729 return ret;
1730 }
1731
1732 int reflink(int infd, int outfd) {
1733 int r;
1734
1735 assert(infd >= 0);
1736 assert(outfd >= 0);
1737
1738 /* Make sure we invoke the ioctl on a regular file, so that no device driver accidentally gets it. */
1739
1740 r = fd_verify_regular(outfd);
1741 if (r < 0)
1742 return r;
1743
1744 /* FICLONE was introduced in Linux 4.5 but it uses the same number as BTRFS_IOC_CLONE introduced earlier */
1745
1746 assert_cc(FICLONE == BTRFS_IOC_CLONE);
1747
1748 return RET_NERRNO(ioctl(outfd, FICLONE, infd));
1749 }
1750
1751 assert_cc(sizeof(struct file_clone_range) == sizeof(struct btrfs_ioctl_clone_range_args));
1752
1753 int reflink_range(int infd, uint64_t in_offset, int outfd, uint64_t out_offset, uint64_t sz) {
1754 struct file_clone_range args = {
1755 .src_fd = infd,
1756 .src_offset = in_offset,
1757 .src_length = sz,
1758 .dest_offset = out_offset,
1759 };
1760 int r;
1761
1762 assert(infd >= 0);
1763 assert(outfd >= 0);
1764
1765 /* Inside the kernel, FICLONE is identical to FICLONERANGE with offsets and size set to zero, let's
1766 * simplify things and use the simple ioctl in that case. Also, do the same if the size is
1767 * UINT64_MAX, which is how we usually encode "everything". */
1768 if (in_offset == 0 && out_offset == 0 && IN_SET(sz, 0, UINT64_MAX))
1769 return reflink(infd, outfd);
1770
1771 r = fd_verify_regular(outfd);
1772 if (r < 0)
1773 return r;
1774
1775 assert_cc(FICLONERANGE == BTRFS_IOC_CLONE_RANGE);
1776
1777 return RET_NERRNO(ioctl(outfd, FICLONERANGE, &args));
1778 }