Debian Codegraph @

lsbtr

raw
1 #!/usr/bin/perl
2 #
3 # btrbk - Create snapshots and remote backups of btrfs subvolumes
4 #
5 # Copyright (C) 2014-2022 Axel Burri
6 #
7 # This program is free software: you can redistribute it and/or modify
8 # it under the terms of the GNU General Public License as published by
9 # the Free Software Foundation, either version 3 of the License, or
10 # (at your option) any later version.
11 #
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU General Public License for more details.
16 #
17 # You should have received a copy of the GNU General Public License
18 # along with this program. If not, see <http://www.gnu.org/licenses/>.
19 #
20 # ---------------------------------------------------------------------
21 # The official btrbk website is located at:
22 # https://digint.ch/btrbk/
23 #
24 # Author:
25 # Axel Burri <[email protected]>
26 # ---------------------------------------------------------------------
27
28 use strict;
29 use warnings FATAL => qw( all ), NONFATAL => qw( deprecated );
30
31 use Carp qw(confess);
32 use Getopt::Long qw(GetOptions);
33 use Time::Local qw( timelocal timegm timegm_nocheck );
34 use IPC::Open3 qw(open3);
35 use Symbol qw(gensym);
36 use Cwd qw(abs_path);
37
38 our $VERSION = '0.32.6';
39 our $AUTHOR = 'Axel Burri <[email protected]>';
40 our $PROJECT_HOME = '<https://digint.ch/btrbk/>';
41
42 our $BTRFS_PROGS_MIN = '4.12'; # required since btrbk-v0.27.0
43
44 my $VERSION_INFO = "btrbk command line client, version $VERSION";
45
46
47 my @config_src = ("/etc/btrbk.conf", "/etc/btrbk/btrbk.conf");
48
49 my %compression = (
50 # NOTE: also adapt "compress_list" in ssh_filter_btrbk.sh if you change this
51 gzip => { name => 'gzip', format => 'gz', compress_cmd => [ 'gzip', '-c' ], decompress_cmd => [ 'gzip', '-d', '-c' ], level_min => 1, level_max => 9 },
52 pigz => { name => 'pigz', format => 'gz', compress_cmd => [ 'pigz', '-c' ], decompress_cmd => [ 'pigz', '-d', '-c' ], level_min => 1, level_max => 9, threads => '-p' },
53 bzip2 => { name => 'bzip2', format => 'bz2', compress_cmd => [ 'bzip2', '-c' ], decompress_cmd => [ 'bzip2', '-d', '-c' ], level_min => 1, level_max => 9 },
54 pbzip2 => { name => 'pbzip2', format => 'bz2', compress_cmd => [ 'pbzip2', '-c' ], decompress_cmd => [ 'pbzip2', '-d', '-c' ], level_min => 1, level_max => 9, threads => '-p' },
55 bzip3 => { name => 'bzip3', format => 'bz3', compress_cmd => [ 'bzip3', '-c' ], decompress_cmd => [ 'bzip3', '-d', '-c' ], threads => '-j' },
56 xz => { name => 'xz', format => 'xz', compress_cmd => [ 'xz', '-c' ], decompress_cmd => [ 'xz', '-d', '-c' ], level_min => 0, level_max => 9, threads => '-T' },
57 lzo => { name => 'lzo', format => 'lzo', compress_cmd => [ 'lzop', '-c' ], decompress_cmd => [ 'lzop', '-d', '-c' ], level_min => 1, level_max => 9 },
58 lz4 => { name => 'lz4', format => 'lz4', compress_cmd => [ 'lz4', '-c' ], decompress_cmd => [ 'lz4', '-d', '-c' ], level_min => 1, level_max => 9 },
59 zstd => { name => 'zstd', format => 'zst', compress_cmd => [ 'zstd', '-c' ], decompress_cmd => [ 'zstd', '-d', '-c' ], level_min => 1, level_max => 19, threads => '-T', long => '--long=', adapt => '--adapt' },
60 );
61
62 my $compress_format_alt = join '|', map { $_->{format} } values %compression; # note: this contains duplicate alternations
63 my $ipv4_addr_match = qr/(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])/;
64 my $ipv6_addr_match = qr/[a-fA-F0-9]*:[a-fA-F0-9]*:[a-fA-F0-9:]+/; # simplified (contains at least two colons), matches "::1", "2001:db8::7"
65 my $host_name_match = qr/(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9])/;
66 my $uuid_match = qr/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/;
67 my $btrbk_timestamp_match = qr/(?<YYYY>[0-9]{4})(?<MM>[0-9]{2})(?<DD>[0-9]{2})(T(?<hh>[0-9]{2})(?<mm>[0-9]{2})((?<ss>[0-9]{2})(?<zz>(Z|[+-][0-9]{4})))?)?(_(?<NN>[0-9]+))?/; # matches "YYYYMMDD[Thhmm[ss+0000]][_NN]"
68 my $raw_postfix_match = qr/\.btrfs(\.($compress_format_alt))?(\.(gpg|encrypted))?/; # matches ".btrfs[.gz|.bz2|.xz|...][.gpg|.encrypted]"
69 my $safe_file_match = qr/[0-9a-zA-Z_@\+\-\.\/]+/; # note: ubuntu uses '@' in the subvolume layout: <https://help.ubuntu.com/community/btrfs>
70
71 my $group_match = qr/[a-zA-Z0-9_:-]+/;
72 my $config_split_match = qr/\s*[,\s]\s*/;
73
74 my %day_of_week_map = ( sunday => 0, monday => 1, tuesday => 2, wednesday => 3, thursday => 4, friday => 5, saturday => 6 );
75 my @syslog_facilities = qw( user mail daemon auth lpr news cron authpriv local0 local1 local2 local3 local4 local5 local6 local7 );
76
77 my @incremental_prefs_avail = qw(sro srn sao san aro arn);
78 my @incremental_prefs_default = qw(sro:1 srn:1 sao:1 san:1 aro:1 arn:1);
79 my $incremental_prefs_match = "(defaults|(" . join("|", @incremental_prefs_avail) . ")(:[0-9]+)?)";
80
81 my %config_options = (
82 # NOTE: the parser always maps "no" to undef
83 # NOTE: keys "volume", "subvolume" and "target" are hardcoded
84 # NOTE: files "." and "no" map to <undef>
85 timestamp_format => { default => "long", accept => [qw( short long long-iso )], context => [qw( global volume subvolume )] },
86 snapshot_dir => { default => undef, accept_file => { relative => 1, absolute => 1 }, context => [qw( global volume subvolume )] },
87 snapshot_name => { c_default => 1, accept_file => { name_only => 1 }, context => [qw( subvolume )], deny_glob_context => 1 }, # NOTE: defaults to the subvolume name (hardcoded)
88 snapshot_create => { default => "always", accept => [qw( no always ondemand onchange )], context => [qw( global volume subvolume )] },
89 incremental => { default => "yes", accept => [qw( yes no strict )] },
90 incremental_prefs => { default => \@incremental_prefs_default, accept => [ qr/$incremental_prefs_match/ ], split => 1 },
91 incremental_clones => { default => "yes", accept => [qw( yes no )] },
92 incremental_resolve => { default => "mountpoint", accept => [qw( mountpoint directory _all_accessible )] },
93 preserve_day_of_week => { default => "sunday", accept => [ (keys %day_of_week_map) ] },
94 preserve_hour_of_day => { default => 0, accept => [ (0..23) ] },
95 snapshot_preserve => { default => undef, accept => [qw( no )], accept_preserve_matrix => 1, context => [qw( global volume subvolume )], },
96 snapshot_preserve_min => { default => "all", accept => [qw( all latest ), qr/[1-9][0-9]*[hdwmy]/ ], context => [qw( global volume subvolume )], },
97 target_preserve => { default => undef, accept => [qw( no )], accept_preserve_matrix => 1 },
98 target_preserve_min => { default => "all", accept => [qw( all latest no ), qr/[0-9]+[hdwmy]/ ] },
99 archive_preserve => { default => undef, accept => [qw( no )], accept_preserve_matrix => 1, context => [qw( global )] },
100 archive_preserve_min => { default => "all", accept => [qw( all latest no ), qr/[0-9]+[hdwmy]/ ], context => [qw( global )] },
101 ssh_identity => { default => undef, accept => [qw( no ) ], accept_file => { absolute => 1 } },
102 ssh_user => { default => "root", accept => [qw( no ), qr/[a-z_][a-z0-9_-]*/ ] },
103 ssh_compression => { default => undef, accept => [qw( yes no )] },
104 ssh_cipher_spec => { default => [ "default" ], accept => [qw( default ), qr/[a-z0-9][[email protected]]+/ ], split => 1 },
105 transaction_log => { default => undef, accept => [qw( no )], accept_file => { absolute => 1 }, context => [qw( global )] },
106 transaction_syslog => { default => undef, accept => [qw( no ), @syslog_facilities ], context => [qw( global )] },
107 lockfile => { default => undef, accept => [qw( no )], accept_file => { absolute => 1 }, context => [qw( global )] },
108
109 rate_limit => { default => undef, accept => [qw( no ), qr/[0-9]+[kmgtKMGT]?/ ], require_bin => 'mbuffer' },
110 rate_limit_remote => { default => undef, accept => [qw( no ), qr/[0-9]+[kmgtKMGT]?/ ] }, # NOTE: requires 'mbuffer' command on remote hosts
111 stream_buffer => { default => undef, accept => [qw( no ), qr/[0-9]+[kmgKMG%]?/ ], require_bin => 'mbuffer' },
112 stream_buffer_remote => { default => undef, accept => [qw( no ), qr/[0-9]+[kmgKMG%]?/ ] }, # NOTE: requires 'mbuffer' command on remote hosts
113 stream_compress => { default => undef, accept => [qw( no ), (keys %compression) ] },
114 stream_compress_level => { default => "default", accept => [qw( default ), qr/[0-9]+/ ] },
115 stream_compress_long => { default => "default", accept => [qw( default ), qr/[0-9]+/ ] },
116 stream_compress_threads => { default => "default", accept => [qw( default ), qr/[0-9]+/ ] },
117 stream_compress_adapt => { default => undef, accept => [qw( yes no )] },
118
119 raw_target_compress => { default => undef, accept => [qw( no ), (keys %compression) ] },
120 raw_target_compress_level => { default => "default", accept => [qw( default ), qr/[0-9]+/ ] },
121 raw_target_compress_long => { default => "default", accept => [qw( default ), qr/[0-9]+/ ] },
122 raw_target_compress_threads => { default => "default", accept => [qw( default ), qr/[0-9]+/ ] },
123 raw_target_encrypt => { default => undef, accept => [qw( no gpg openssl_enc )] },
124 raw_target_block_size => { default => "128K", accept => [ qr/[0-9]+[kmgKMG]?/ ] },
125 raw_target_split => { default => undef, accept => [qw( no ), qr/[0-9]+([kmgtpezyKMGTPEZY][bB]?)?/ ] },
126 gpg_keyring => { default => undef, accept_file => { absolute => 1 } },
127 gpg_recipient => { default => undef, accept => [ qr/[0-9a-zA-Z_@\+\-\.]+/ ], split => 1 },
128 openssl_ciphername => { default => "aes-256-cbc", accept => [ qr/[0-9a-zA-Z\-]+/ ] },
129 openssl_iv_size => { default => undef, accept => [qw( no ), qr/[0-9]+/ ] },
130 openssl_keyfile => { default => undef, accept_file => { absolute => 1 } },
131
132 kdf_backend => { default => undef, accept_file => { absolute => 1 } },
133 kdf_keysize => { default => "32", accept => [ qr/[0-9]+/ ] },
134 kdf_keygen => { default => "once", accept => [qw( once each )] },
135
136 group => { default => undef, accept => [ qr/$group_match/ ], allow_multiple => 1, split => 1 },
137 noauto => { default => undef, accept => [qw( yes no )] },
138
139 backend => { default => "btrfs-progs", accept => [qw( btrfs-progs btrfs-progs-btrbk btrfs-progs-sudo btrfs-progs-doas )] },
140 backend_local => { default => undef, accept => [qw( no btrfs-progs btrfs-progs-btrbk btrfs-progs-sudo btrfs-progs-doas )] },
141 backend_remote => { default => undef, accept => [qw( no btrfs-progs btrfs-progs-btrbk btrfs-progs-sudo btrfs-progs-doas )] },
142 backend_local_user => { default => undef, accept => [qw( no btrfs-progs btrfs-progs-btrbk btrfs-progs-sudo btrfs-progs-doas )] },
143
144 compat => { default => undef, accept => [qw( no busybox ignore_receive_errors )], split => 1 },
145 compat_local => { default => undef, accept => [qw( no busybox ignore_receive_errors )], split => 1 },
146 compat_remote => { default => undef, accept => [qw( no busybox ignore_receive_errors )], split => 1 },
147 safe_commands => { default => undef, accept => [qw( yes no )], context => [qw( global )] },
148 btrfs_commit_delete => { default => undef, accept => [qw( yes no after each )],
149 deprecated => { MATCH => { regex => qr/^(?:after|each)$/, warn => 'Please use "btrfs_commit_delete yes|no"', replace_key => "btrfs_commit_delete", replace_value => "yes" } } },
150 send_protocol => { default => undef, accept => [qw( no 1 2 )] }, # NOTE: requires btrfs-progs-5.19
151 send_compressed_data => { default => undef, accept => [qw( yes no )] }, # NOTE: implies send_protocol=2
152
153 snapshot_qgroup_destroy => { default => undef, accept => [qw( yes no )], context => [qw( global volume subvolume )] },
154 target_qgroup_destroy => { default => undef, accept => [qw( yes no )] },
155 archive_qgroup_destroy => { default => undef, accept => [qw( yes no )], context => [qw( global )] },
156
157 archive_exclude => { default => undef, accept_file => { wildcards => 1 }, allow_multiple => 1, context => [qw( global )] },
158 archive_exclude_older => { default => undef, accept => [qw( yes no )] },
159
160 cache_dir => { default => undef, accept_file => { absolute => 1 }, allow_multiple => 1, context => [qw( global )] },
161 ignore_extent_data_inline => { default => "yes", accept => [qw( yes no )] },
162
163 warn_unknown_targets => { default => undef, accept => [qw( yes no )] },
164
165 # deprecated options
166 ssh_port => { default => "default", accept => [qw( default ), qr/[0-9]+/ ],
167 deprecated => { DEFAULT => { warn => 'Please use "ssh://hostname[:port]" notation in the "volume" and "target" configuration lines.' } } },
168 btrfs_progs_compat => { default => undef, accept => [qw( yes no )],
169 deprecated => { DEFAULT => { ABORT => 1, warn => 'This feature has been dropped in btrbk-v0.23.0. Please update to newest btrfs-progs, AT LEAST >= $BTRFS_PROGS_MIN' } } },
170 snapshot_preserve_daily => { default => 'all', accept => [qw( all ), qr/[0-9]+/ ], context => [qw( global volume subvolume )],
171 deprecated => { DEFAULT => { FAILSAFE_PRESERVE => 1, warn => 'Please use "snapshot_preserve" and/or "snapshot_preserve_min"' } } },
172 snapshot_preserve_weekly => { default => 0, accept => [qw( all ), qr/[0-9]+/ ], context => [qw( global volume subvolume )],
173 deprecated => { DEFAULT => { FAILSAFE_PRESERVE => 1, warn => 'Please use "snapshot_preserve" and/or "snapshot_preserve_min"' } } },
174 snapshot_preserve_monthly => { default => 'all', accept => [qw( all ), qr/[0-9]+/ ], context => [qw( global volume subvolume )],
175 deprecated => { DEFAULT => { FAILSAFE_PRESERVE => 1, warn => 'Please use "snapshot_preserve" and/or "snapshot_preserve_min"' } } },
176 target_preserve_daily => { default => 'all', accept => [qw( all ), qr/[0-9]+/ ],
177 deprecated => { DEFAULT => { FAILSAFE_PRESERVE => 1, warn => 'Please use "target_preserve" and/or "target_preserve_min"' } } },
178 target_preserve_weekly => { default => 0, accept => [qw( all ), qr/[0-9]+/ ],
179 deprecated => { DEFAULT => { FAILSAFE_PRESERVE => 1, warn => 'Please use "target_preserve" and/or "target_preserve_min"' } } },
180 target_preserve_monthly => { default => 'all', accept => [qw( all ), qr/[0-9]+/ ],
181 deprecated => { DEFAULT => { FAILSAFE_PRESERVE => 1, warn => 'Please use "target_preserve" and/or "target_preserve_min"' } } },
182 resume_missing => { default => "yes", accept => [qw( yes no )],
183 deprecated => { yes => { warn => 'ignoring (missing backups are always resumed since btrbk v0.23.0)' },
184 no => { FAILSAFE_PRESERVE => 1, warn => 'Please use "target_preserve_min latest" and "target_preserve no" if you want to keep only the latest backup', },
185 DEFAULT => {} } },
186 snapshot_create_always => { default => undef, accept => [qw( yes no )],
187 deprecated => { yes => { warn => "Please use \"snapshot_create always\"",
188 replace_key => "snapshot_create",
189 replace_value => "always",
190 },
191 no => { warn => "Please use \"snapshot_create no\" or \"snapshot_create ondemand\"",
192 replace_key => "snapshot_create",
193 replace_value => "ondemand",
194 },
195 DEFAULT => {},
196 },
197 },
198 receive_log => { default => undef, accept => [qw( sidecar no )], accept_file => { absolute => 1 },
199 deprecated => { DEFAULT => { warn => "ignoring" } },
200 }
201 );
202
203 my @config_target_types = qw(send-receive raw); # first in list is default
204
205 my %table_formats = (
206 config_volume => {
207 table => [ qw( -volume_host -volume_port volume_path ) ],
208 long => [ qw( volume_host -volume_port volume_path -volume_rsh ) ],
209 raw => [ qw( volume_url volume_host volume_port volume_path volume_rsh ) ],
210 single_column => [ qw( volume_url ) ],
211 },
212
213 config_source => {
214 table => [ qw( -source_host -source_port source_subvolume snapshot_path snapshot_name ) ],
215 long => [ qw( source_host -source_port source_subvolume snapshot_path snapshot_name -source_rsh ) ],
216 raw => [ qw( source_url source_host source_port source_subvolume snapshot_path snapshot_name source_rsh ) ],
217 single_column => [ qw( source_url ) ],
218 },
219
220 config_target => {
221 table => [ qw( -target_host -target_port target_path ) ],
222 long => [ qw( target_host -target_port target_path -target_rsh ) ],
223 raw => [ qw( target_url target_host target_port target_path target_rsh ) ],
224 single_column => [ qw( target_url ) ],
225 },
226
227 config => {
228 table => [ qw( -source_host -source_port source_subvolume snapshot_path snapshot_name -target_host -target_port target_path ) ],
229 long => [ qw( -source_host -source_port source_subvolume snapshot_path snapshot_name -target_host -target_port target_path target_type snapshot_preserve target_preserve ) ],
230 raw => [ qw( source_url source_host source_port source_subvolume snapshot_path snapshot_name target_url target_host target_port target_path target_type snapshot_preserve target_preserve source_rsh target_rsh ) ],
231 },
232
233 resolved => {
234 table => [ qw( -source_host -source_port source_subvolume snapshot_subvolume status -target_host -target_port target_subvolume ) ],
235 long => [ qw( -source_host -source_port source_subvolume snapshot_subvolume status -target_host -target_port target_subvolume target_type ) ],
236 raw => [ qw( type source_url source_host source_port source_subvolume snapshot_subvolume snapshot_name status target_url target_host target_port target_subvolume target_type source_rsh target_rsh ) ],
237 },
238
239 snapshots => {
240 table => [ qw( -source_host -source_port source_subvolume snapshot_subvolume status ) ],
241 long => [ qw( -source_host -source_port source_subvolume snapshot_subvolume status ) ],
242 raw => [ qw( source_url source_host source_port source_subvolume snapshot_subvolume snapshot_name status source_rsh ) ],
243 single_column => [ qw( snapshot_url ) ],
244 },
245
246 backups => { # same as resolved, except for single_column
247 table => [ qw( -source_host -source_port source_subvolume snapshot_subvolume status -target_host -target_port target_subvolume ) ],
248 long => [ qw( -source_host -source_port source_subvolume snapshot_subvolume status -target_host -target_port target_subvolume target_type ) ],
249 raw => [ qw( type source_url source_host source_port source_subvolume snapshot_subvolume snapshot_name status target_url target_host target_port target_subvolume target_type source_rsh target_rsh ) ],
250 single_column => [ qw( target_url ) ],
251 },
252
253 latest => { # same as resolved, except hiding target if not present
254 table => [ qw( -source_host -source_port source_subvolume snapshot_subvolume status -target_host -target_port -target_subvolume ) ],
255 long => [ qw( -source_host -source_port source_subvolume snapshot_subvolume status -target_host -target_port -target_subvolume -target_type ) ],
256 raw => [ qw( type source_url source_host source_port source_subvolume snapshot_subvolume snapshot_name status target_url target_host target_port target_subvolume target_type source_rsh target_rsh ) ],
257 },
258
259 stats => {
260 table => [ qw( -source_host -source_port source_subvolume snapshot_subvolume -target_host -target_port -target_subvolume snapshots -backups ) ],
261 long => [ qw( -source_host -source_port source_subvolume snapshot_subvolume -target_host -target_port -target_subvolume snapshot_status backup_status snapshots -backups -correlated -orphaned -incomplete ) ],
262 raw => [ qw( source_url source_host source_port source_subvolume snapshot_subvolume snapshot_name target_url target_host target_port target_subvolume snapshot_status backup_status snapshots backups correlated orphaned incomplete ) ],
263 RALIGN => { snapshots=>1, backups=>1, correlated=>1, orphaned=>1, incomplete=>1 },
264 },
265
266 schedule => {
267 table => [ qw( action -host -port subvolume scheme reason ) ],
268 long => [ qw( action -host -port subvolume scheme reason ) ],
269 raw => [ qw( topic action url host port path hod dow min h d w m y) ],
270 },
271
272 usage => {
273 table => [ qw( -host -port mount_source path size used free ) ],
274 long => [ qw( type -host -port mount_source path size used device_size device_allocated device_unallocated device_missing device_used free free_min data_ratio metadata_ratio global_reserve global_reserve_used ) ],
275 raw => [ qw( type host port mount_source path size used device_size device_allocated device_unallocated device_missing device_used free free_min data_ratio metadata_ratio global_reserve global_reserve_used ) ],
276 RALIGN => { size=>1, used=>1, device_size=>1, device_allocated=>1, device_unallocated=>1, device_missing=>1, device_used=>1, free=>1, free_min=>1, data_ratio=>1, metadata_ratio=>1, global_reserve=>1, global_reserve_used=>1 },
277 },
278
279 transaction => {
280 table => [ qw( type status -target_host -target_port target_subvolume -source_host -source_port source_subvolume parent_subvolume ) ],
281 long => [ qw( localtime type status duration target_host -target_port target_subvolume source_host -source_port source_subvolume parent_subvolume message ) ],
282 tlog => [ qw( localtime type status target_url source_url parent_url message ) ],
283 syslog => [ qw( type status target_url source_url parent_url message ) ],
284 raw => [ qw( time localtime type status duration target_url source_url parent_url message ) ],
285 },
286
287 origin_tree => {
288 table => [ qw( tree uuid parent_uuid received_uuid ) ],
289 long => [ qw( tree uuid parent_uuid received_uuid recursion ) ],
290 raw => [ qw( tree uuid parent_uuid received_uuid recursion ) ],
291 },
292
293 diff => {
294 table => [ qw( flags count size file ) ],
295 long => [ qw( flags count size file ) ],
296 raw => [ qw( flags count size file ) ],
297 RALIGN => { count=>1, size=>1 },
298 },
299
300 fs_list => {
301 table => [ qw( -host mount_source mount_subvol mount_point id flags subvolume_path path ) ],
302 short => [ qw( -host mount_source id flags path ) ],
303 long => [ qw( -host mount_source id top cgen gen uuid parent_uuid received_uuid flags path ) ],
304 raw => [ qw( host mount_source mount_subvol mount_point mount_subvolid id top_level cgen gen uuid parent_uuid received_uuid readonly path subvolume_path subvolume_rel_path url ) ],
305 single_column => [ qw( url ) ],
306 RALIGN => { id=>1, top=>1, cgen=>1, gen=>1 },
307 },
308
309 extent_diff => {
310 table => [ qw( total exclusive -diff -set subvol ) ],
311 long => [ qw( id cgen gen total exclusive -diff -set subvol ) ],
312 raw => [ qw( id cgen gen total exclusive -diff -set subvol ) ],
313 RALIGN => { id=>1, cgen=>1, gen=>1, total=>1, exclusive=>1, diff=>1, set=>1 },
314 },
315 );
316
317 my @btrfs_cmd = (
318 "btrfs subvolume list",
319 "btrfs subvolume show",
320 "btrfs subvolume snapshot",
321 "btrfs subvolume delete",
322 "btrfs send",
323 "btrfs receive",
324 "btrfs filesystem usage",
325 "btrfs qgroup destroy",
326 );
327 my @system_cmd = (
328 "readlink",
329 "test",
330 );
331 my %backend_cmd_map = (
332 "btrfs-progs-btrbk" => { map +( $_ => [ s/ /-/gr ] ), @btrfs_cmd },
333 "btrfs-progs-sudo" => { map +( $_ => [ qw( sudo -n ), split(" ", $_) ] ), @btrfs_cmd, @system_cmd },
334 "btrfs-progs-doas" => { map +( $_ => [ qw( doas -n ), split(" ", $_) ] ), @btrfs_cmd, @system_cmd },
335 );
336
337 # keys used in raw target sidecar files (.info):
338 my %raw_info_sort = (
339 TYPE => 1,
340 FILE => 2, # informative only (as of btrbk-0.32.6)
341 RECEIVED_UUID => 3,
342 RECEIVED_PARENT_UUID => 4,
343 compress => 10,
344 split => 11,
345 encrypt => 12,
346 cipher => 13,
347 iv => 14,
348 # kdf_* (generated by kdf_backend)
349 INCOMPLETE => 100,
350 );
351
352 my $raw_info_value_match = qr/[0-9a-zA-Z_-]*/;
353
354 my %raw_url_cache; # map URL to (fake) btr_tree node
355 my %mountinfo_cache; # map MACHINE_ID to mount points (sorted descending by file length)
356 my %mount_source_cache; # map URL_PREFIX:mount_source (aka device) to btr_tree node
357 my %uuid_cache; # map UUID to btr_tree node
358 my %realpath_cache; # map URL to realpath (symlink target). empty string denotes an error.
359
360 my $tree_inject_id = 0; # fake subvolume id for injected nodes (negative)
361 my $fake_uuid_prefix = 'XXXXXXXX-XXXX-XXXX-XXXX-'; # plus 0-padded inject_id: XXXXXXXX-XXXX-XXXX-XXXX-000000000000
362
363 my $program_name; # "btrbk" or "lsbtr", default to "btrbk"
364 my $safe_commands;
365 my $dryrun;
366 my $loglevel = 1;
367 my $quiet;
368 my @exclude_vf;
369 my $do_dumper;
370 my $do_trace;
371 my $show_progress = 0;
372 my $output_format;
373 my $output_pretty = 0;
374 my @output_unit;
375 my $lockfile;
376 my $tlog_fh;
377 my $syslog_enabled = 0;
378 my $current_transaction;
379 my @transaction_log;
380 my %config_override;
381 my @tm_now; # current localtime ( sec, min, hour, mday, mon, year, wday, yday, isdst )
382 my @stderr; # stderr of last run_cmd
383 my %warn_once;
384 my %kdf_vars;
385 my $kdf_session_key;
386
387
388 $SIG{__DIE__} = sub {
389 print STDERR "\nERROR: process died unexpectedly (btrbk v$VERSION)";
390 print STDERR "\nPlease contact the author: $AUTHOR\n\n";
391 print STDERR "Stack Trace:\n----------------------------------------\n";
392 Carp::confess @_;
393 };
394
395 $SIG{INT} = sub {
396 print STDERR "\nERROR: Caught SIGINT, dumping transaction log:\n";
397 action("signal", status => "SIGINT");
398 print_formatted("transaction", \@transaction_log, output_format => "tlog", outfile => *STDERR);
399 exit 1;
400 };
401
402 sub VERSION_MESSAGE
403 {
404 print $VERSION_INFO . "\n";
405 }
406
407 sub ERROR_HELP_MESSAGE
408 {
409 return if($quiet);
410 print STDERR "See '$program_name --help'.\n";
411 }
412
413 sub HELP_MESSAGE
414 {
415 return if($quiet);
416 #80-----------------------------------------------------------------------------
417 if($program_name eq "lsbtr") {
418 print <<"END_HELP_LSBTR";
419 usage: lsbtr [<options>] [[--] <path>...]
420
421 options:
422 -h, --help display this help message
423 --version display version information
424 -l, --long use long listing format
425 -u, --uuid print uuid table (parent/received relations)
426 -1, --single-column Print path column only
427 --raw print raw table format
428 -v, --verbose increase output verbosity
429 -c, --config=FILE specify btrbk configuration file
430 --override=KEY=VALUE globally override a configuration option
431
432 For additional information, see $PROJECT_HOME
433 END_HELP_LSBTR
434 }
435 else {
436 print <<"END_HELP_BTRBK";
437 usage: btrbk [<options>] <command> [[--] <filter>...]
438
439 options:
440 -h, --help display this help message
441 --version display version information
442 -c, --config=FILE specify configuration file
443 -n, --dry-run perform a trial run with no changes made
444 --exclude=FILTER exclude configured sections
445 -p, --preserve preserve all (do not delete anything)
446 --preserve-snapshots preserve snapshots (do not delete snapshots)
447 --preserve-backups preserve backups (do not delete backups)
448 --wipe delete all but latest snapshots
449 -v, --verbose be more verbose (increase logging level)
450 -q, --quiet be quiet (do not print backup summary)
451 -l, --loglevel=LEVEL set logging level (error, warn, info, debug, trace)
452 -t, --table change output to table format
453 -L, --long change output to long format
454 --format=FORMAT change output format, FORMAT=table|long|raw
455 -S, --print-schedule print scheduler details (for the "run" command)
456 --progress show progress bar on send-receive operation
457 --lockfile=FILE create and check lockfile
458 --override=KEY=VALUE globally override a configuration option
459
460 commands:
461 run run snapshot and backup operations
462 dryrun don't run btrfs commands; show what would be executed
463 snapshot run snapshot operations only
464 resume run backup operations, and delete snapshots
465 prune only delete snapshots and backups
466 archive <src> <dst> recursively copy all subvolumes
467 clean delete incomplete (garbled) backups
468 stats print snapshot/backup statistics
469 list <subcommand> available subcommands are (default "all"):
470 all snapshots and backups
471 snapshots snapshots
472 backups backups and correlated snapshots
473 latest most recent snapshots and backups
474 config configured source/snapshot/target relations
475 source configured source/snapshot relations
476 volume configured volume sections
477 target configured targets
478 usage print filesystem usage
479 ls <path> list all btrfs subvolumes below path
480 origin <subvol> print origin information for subvolume
481 diff <from> <to> list file changes between related subvolumes
482 extents [diff] <path> calculate accurate disk space usage
483
484 For additional information, see $PROJECT_HOME
485 END_HELP_BTRBK
486 }
487 #80-----------------------------------------------------------------------------
488 }
489
490 sub _log_cont { my $p = shift; print STDERR $p . join("\n${p}... ", grep defined, @_) . "\n"; }
491 sub TRACE { print STDERR map { "___ $_\n" } @_ if($loglevel >= 4) }
492 sub DEBUG { _log_cont("", @_) if($loglevel >= 3) }
493 sub INFO { _log_cont("", @_) if($loglevel >= 2) }
494 sub WARN { _log_cont("WARNING: ", @_) if($loglevel >= 1) }
495 sub ERROR { _log_cont("ERROR: ", @_) }
496
497 sub INFO_ONCE {
498 my $t = shift;
499 if($warn_once{INFO}{$t}) { TRACE("INFO(again): $t", @_) if($do_trace); return 0; }
500 else { $warn_once{INFO}{$t} = 1; INFO($t, @_); return 1; }
501 }
502 sub WARN_ONCE {
503 my $t = shift;
504 if($warn_once{WARN}{$t}) { TRACE("WARNING(again): $t", @_) if($do_trace); return 0; }
505 else { $warn_once{WARN}{$t} = 1; WARN($t, @_); return 1; }
506 }
507
508 sub VINFO {
509 return undef unless($do_dumper);
510 my $vinfo = shift; my $t = shift || "vinfo"; my $maxdepth = shift // 2;
511 print STDERR Data::Dumper->new([$vinfo], [$t])->Maxdepth($maxdepth)->Dump();
512 }
513 sub SUBVOL_LIST {
514 return undef unless($do_dumper);
515 my $vol = shift; my $t = shift // "SUBVOL_LIST"; my $svl = vinfo_subvol_list($vol);
516 print STDERR "$t:\n " . join("\n ", map { "$vol->{PRINT}/./$_->{SUBVOL_PATH}\t$_->{node}{id}" } @$svl) . "\n";
517 }
518
519
520 sub ABORTED($$;$)
521 {
522 my $config = shift;
523 my $abrt_key = shift // die;
524 my $abrt = shift;
525 $config = $config->{CONFIG} if($config->{CONFIG}); # accept vinfo for $config
526 unless(defined($abrt)) {
527 # no key (only text) set: switch arguments, use default key
528 $abrt = $abrt_key;
529 $abrt_key = "abort_" . $config->{CONTEXT};
530 }
531 unless($abrt_key =~ /^skip_/) {
532 # keys starting with "skip_" are not actions
533 $abrt =~ s/\n/\\\\/g;
534 $abrt =~ s/\r//g;
535 action($abrt_key,
536 status => "ABORT",
537 vinfo_prefixed_keys("target", vinfo($config->{url}, $config)),
538 message => $abrt,
539 );
540 }
541 $config->{ABORTED} = { key => $abrt_key, text => $abrt };
542 }
543
544 sub IS_ABORTED($;$)
545 {
546 my $config = shift;
547 $config = $config->{CONFIG} if($config->{CONFIG}); # accept vinfo for $config
548 return undef unless(defined($config->{ABORTED}));
549 my $abrt_key = $config->{ABORTED}->{key};
550 return undef unless(defined($abrt_key));
551 my $filter_prefix = shift;
552 return undef if($filter_prefix && ($abrt_key !~ /^$filter_prefix/));
553 return $abrt_key;
554 }
555
556 sub ABORTED_TEXT($)
557 {
558 my $config = shift;
559 $config = $config->{CONFIG} if($config->{CONFIG}); # accept vinfo for $config
560 return "" unless(defined($config->{ABORTED}));
561 return $config->{ABORTED}->{text} // "";
562 }
563
564 sub FIX_MANUALLY($$)
565 {
566 # treated as error, but does not abort config section
567 my $config = shift;
568 $config = $config->{CONFIG} if($config->{CONFIG}); # accept vinfo for $config
569 my $msg = shift // die;
570 $config->{FIX_MANUALLY} //= [];
571 push(@{$config->{FIX_MANUALLY}}, $msg);
572 }
573
574 sub eval_quiet(&)
575 {
576 local $SIG{__DIE__};
577 return eval { $_[0]->() }
578 }
579
580 sub require_data_dumper
581 {
582 if(eval_quiet { require Data::Dumper; }) {
583 Data::Dumper->import("Dumper");
584 $Data::Dumper::Sortkeys = 1;
585 $Data::Dumper::Quotekeys = 0;
586 $do_dumper = 1;
587 # silence perl warning: Name "Data::Dumper::Sortkeys" used only once: possible typo at...
588 TRACE "Successfully loaded Dumper module: sortkeys=$Data::Dumper::Sortkeys, quotekeys=$Data::Dumper::Quotekeys" if($do_trace);
589 } else {
590 WARN "Perl module \"Data::Dumper\" not found: data trace dumps disabled!" if($do_trace);
591 }
592 }
593
594 sub init_transaction_log($$)
595 {
596 my $file = shift;
597 my $config_syslog_facility = shift;
598 if(defined($file) && (not $dryrun)) {
599 if(open($tlog_fh, '>>', $file)) {
600 # print headers (disabled)
601 # print_formatted("transaction", [ ], output_format => "tlog", outfile => $tlog_fh);
602 INFO "Using transaction log: $file";
603 } else {
604 $tlog_fh = undef;
605 ERROR "Failed to open transaction log '$file': $!";
606 }
607 }
608 if(defined($config_syslog_facility) && (not $dryrun)) {
609 DEBUG "Opening syslog";
610 if(eval_quiet { require Sys::Syslog; }) {
611 $syslog_enabled = 1;
612 Sys::Syslog::openlog("btrbk", "", $config_syslog_facility);
613 DEBUG "Syslog enabled";
614 }
615 else {
616 WARN "Syslog disabled: $@";
617 }
618 }
619 action("DEFERRED", %$_) foreach (@transaction_log);
620 }
621
622 sub close_transaction_log()
623 {
624 if($tlog_fh) {
625 DEBUG "Closing transaction log";
626 close $tlog_fh || ERROR "Failed to close transaction log: $!";
627 }
628 if($syslog_enabled) {
629 DEBUG "Closing syslog";
630 eval_quiet { Sys::Syslog::closelog(); };
631 }
632 }
633
634 sub action($@)
635 {
636 my $type = shift // die;
637 my $h = { @_ };
638 unless($type eq "DEFERRED") {
639 my $time = $h->{time} // time;
640 $h->{type} = $type;
641 $h->{time} = $time;
642 $h->{localtime} = timestamp($time, 'debug-iso');
643 push @transaction_log, $h;
644 }
645 print_formatted("transaction", [ $h ], output_format => "tlog", no_header => 1, outfile => $tlog_fh) if($tlog_fh);
646 print_formatted("transaction", [ $h ], output_format => "syslog", no_header => 1) if($syslog_enabled); # dirty hack, this calls syslog()
647 return $h;
648 }
649
650 sub start_transaction($@)
651 {
652 my $type = shift // die;
653 my $time = time;
654 die("start_transaction() while transaction is running") if($current_transaction);
655 my @actions = (ref($_[0]) eq "HASH") ? @_ : { @_ }; # single action is not hashref
656 $current_transaction = [];
657 foreach (@actions) {
658 push @$current_transaction, action($type, %$_, status => ($dryrun ? "dryrun_starting" : "starting"), time => $time);
659 }
660 }
661
662 sub end_transaction($$)
663 {
664 my $type = shift // die;
665 my $success = shift; # scalar or coderef: if scalar, status is set for all current transitions
666 my $time = time;
667 die("end_transaction() while no transaction is running") unless($current_transaction);
668 foreach (@$current_transaction) {
669 die("end_transaction() has different type") unless($_->{type} eq $type);
670 my $status = (ref($success) ? &{$success} ($_) : $success) ? "success" : "ERROR";
671 $status = "dryrun_" . $status if($dryrun);
672 action($type, %$_, status => $status, time => $time, duration => ($dryrun ? undef : ($time - $_->{time})));
673 }
674 $current_transaction = undef;
675 }
676
677 sub syslog($)
678 {
679 return undef unless($syslog_enabled);
680 my $line = shift;
681 eval_quiet { Sys::Syslog::syslog("info", $line); };
682 }
683
684 sub check_exe($)
685 {
686 my $cmd = shift // die;
687 foreach my $path (split(":", $ENV{PATH})) {
688 return 1 if( -x "$path/$cmd" );
689 }
690 return 0;
691 }
692
693 sub stream_buffer_cmd_text($)
694 {
695 my $opts = shift;
696 my $rl_in = $opts->{rate_limit_in} // $opts->{rate_limit}; # maximum read rate: b,k,M,G
697 my $rl_out = $opts->{rate_limit_out}; # maximum write rate: b,k,M,G
698 my $bufsize = $opts->{stream_buffer}; # b,k,M,G,% (default: 2%)
699 my $blocksize = $opts->{blocksize}; # defaults to 10k
700 my $progress = $opts->{show_progress};
701
702 # return empty array if mbuffer is not needed
703 return () unless($rl_in || $rl_out || $bufsize || $progress);
704
705 # NOTE: mbuffer takes defaults from /etc/mbuffer.rc
706 my @cmd = ( "mbuffer" );
707 push @cmd, ( "-v", "1" ); # disable warnings (they arrive asynchronously and cant be caught)
708 push @cmd, "-q" unless($progress);
709 push @cmd, ( "-s", $blocksize ) if($blocksize);
710 push @cmd, ( "-m", lc($bufsize) ) if($bufsize);
711 push @cmd, ( "-r", lc($rl_in) ) if($rl_in);
712 push @cmd, ( "-R", lc($rl_out) ) if($rl_out);
713 return { cmd_text => join(' ', @cmd) };
714 }
715
716 sub compress_cmd_text($;$)
717 {
718 my $def = shift // die;
719 my $decompress = shift;
720 my $cc = $compression{$def->{key}};
721 my @cmd = $decompress ? @{$cc->{decompress_cmd}} : @{$cc->{compress_cmd}};
722
723 if((not $decompress) && defined($def->{level}) && ($def->{level} ne "default")) {
724 my $level = $def->{level};
725 if(!defined($cc->{level_min})) {
726 WARN_ONCE "Compression level is not supported for '$cc->{name}', ignoring";
727 $level = undef;
728 } elsif($level < $cc->{level_min}) {
729 WARN_ONCE "Compression level capped to minimum for '$cc->{name}': $cc->{level_min}";
730 $level = $cc->{level_min};
731 } elsif($level > $cc->{level_max}) {
732 WARN_ONCE "Compression level capped to maximum for '$cc->{name}': $cc->{level_max}";
733 $level = $cc->{level_max};
734 }
735 push @cmd, '-' . $level if(defined($level));
736 }
737 if(defined($def->{threads}) && ($def->{threads} ne "default")) {
738 my $thread_opt = $cc->{threads};
739 if($thread_opt) {
740 push @cmd, $thread_opt . $def->{threads};
741 }
742 else {
743 WARN_ONCE "Threading is not supported for '$cc->{name}', ignoring";
744 }
745 }
746 if(defined($def->{long}) && ($def->{long} ne "default")) {
747 my $long_opt = $cc->{long};
748 if($long_opt) {
749 push @cmd, $long_opt . $def->{long};
750 }
751 else {
752 WARN_ONCE "Long distance matching is not supported for '$cc->{name}', ignoring";
753 }
754 }
755 if(defined($def->{adapt})) {
756 my $adapt_opt = $cc->{adapt};
757 if($adapt_opt) {
758 push @cmd, $adapt_opt;
759 }
760 else {
761 WARN_ONCE "Adaptive compression is not supported for '$cc->{name}', ignoring";
762 }
763 }
764 return { cmd_text => join(' ', @cmd) };
765 }
766
767 sub decompress_cmd_text($)
768 {
769 return compress_cmd_text($_[0], 1);
770 }
771
772 sub _piped_cmd_txt($)
773 {
774 my $cmd_pipe = shift;
775 my $cmd = "";
776 my $pipe = "";
777 my $last;
778 foreach (map $_->{cmd_text}, @$cmd_pipe) {
779 die if($last);
780 if(/^>/) {
781 # can't be first, must be last
782 die unless($pipe);
783 $last = 1;
784 $pipe = ' ';
785 }
786 $cmd .= $pipe . $_;
787 $pipe = ' | ';
788 }
789 return $cmd;
790 }
791
792 sub quoteshell(@) {
793 # replace ' -> '\''
794 join ' ', map { "'" . s/'/'\\''/gr . "'" } @_
795 }
796
797 sub _safe_cmd($;$)
798 {
799 # hashes of form: "{ unsafe => 'string' }" get translated to "'string'"
800 my $aref = shift;
801 my $offending = shift;
802 return join ' ', map {
803 if(ref($_)) {
804 my $prefix = $_->{prefix} // "";
805 my $postfix = $_->{postfix} // "";
806 $_ = $_->{unsafe};
807 die "cannot quote leading dash for command: $_" if(/^-/);
808 # NOTE: all files must be absolute
809 if($offending) {
810 push @$offending, $_ unless(defined(check_file($_, { absolute => 1 })));
811 }
812 $_ = $prefix . quoteshell($_) . $postfix;
813 }
814 $_
815 } @$aref;
816 }
817
818 sub run_cmd(@)
819 {
820 # IPC::Open3 based implementation.
821 # NOTE: multiple filters are not supported!
822
823 my @cmd_pipe_in = (ref($_[0]) eq "HASH") ? @_ : { @_ };
824 die unless(scalar(@cmd_pipe_in));
825 @stderr = ();
826
827 my $destructive = 0;
828 my @cmd_pipe;
829 my @unsafe_cmd;
830 my $compressed = undef;
831 my $large_output;
832 my $stream_options = $cmd_pipe_in[0]->{stream_options} // {};
833 my @filter_stderr;
834 my $fatal_stderr;
835 my $has_rsh;
836
837 $cmd_pipe_in[0]->{stream_source} = 1;
838 $cmd_pipe_in[-1]->{stream_sink} = 1;
839
840 foreach my $href (@cmd_pipe_in)
841 {
842 die if(defined($href->{cmd_text}));
843
844 push @filter_stderr, ((ref($href->{filter_stderr}) eq "ARRAY") ? @{$href->{filter_stderr}} : $href->{filter_stderr}) if($href->{filter_stderr});
845 $fatal_stderr = $href->{fatal_stderr} if($href->{fatal_stderr});
846 $destructive = 1 unless($href->{non_destructive});
847 $has_rsh = $href->{rsh} if($href->{rsh});
848 $large_output = 1 if($href->{large_output});
849
850 if($href->{redirect_to_file}) {
851 die unless($href->{stream_sink});
852 $href->{cmd_text} = _safe_cmd([ '>', $href->{redirect_to_file} ], \@unsafe_cmd);
853 }
854 elsif($href->{append_to_file}) {
855 die unless($href->{stream_sink});
856 $href->{cmd_text} = _safe_cmd([ '>>', $href->{append_to_file} ], \@unsafe_cmd);
857 }
858 elsif($href->{compress_stdin}) {
859 # does nothing if already compressed correctly by stream_compress
860 if($compressed && ($compression{$compressed->{key}}->{format} ne $compression{$href->{compress_stdin}->{key}}->{format})) {
861 # re-compress with different algorithm
862 push @cmd_pipe, decompress_cmd_text($compressed);
863 $compressed = undef;
864 }
865 unless($compressed) {
866 push @cmd_pipe, compress_cmd_text($href->{compress_stdin});
867 $compressed = $href->{compress_stdin};
868 }
869
870 next;
871 }
872 elsif($href->{cmd}) {
873 $href->{cmd_text} = _safe_cmd($href->{cmd}, \@unsafe_cmd);
874 }
875 return undef unless(defined($href->{cmd_text}));
876
877 my @rsh_compress_in;
878 my @rsh_compress_out;
879 my @decompress_in;
880
881 # input stream compression: local, in front of rsh_cmd_pipe
882 if($href->{rsh} && $stream_options->{stream_compress} && (not $href->{stream_source})) {
883 if($compressed && ($compression{$compressed->{key}}->{format} ne $compression{$stream_options->{stream_compress}->{key}}->{format})) {
884 # re-compress with different algorithm, should be avoided!
885 push @rsh_compress_in, decompress_cmd_text($compressed);
886 $compressed = undef;
887 }
888 if(not $compressed) {
889 $compressed = $stream_options->{stream_compress};
890 push @rsh_compress_in, compress_cmd_text($compressed);
891 }
892 }
893
894 if($compressed && (not ($href->{compressed_ok}))) {
895 push @decompress_in, decompress_cmd_text($compressed);
896 $compressed = undef;
897 }
898
899 # output stream compression: remote, at end of rsh_cmd_pipe
900 if($href->{rsh} && $stream_options->{stream_compress} && (not $href->{stream_sink}) && (not $compressed)) {
901 $compressed = $stream_options->{stream_compress};
902 push @rsh_compress_out, compress_cmd_text($compressed);
903 }
904
905 if($href->{rsh}) {
906 # honor stream_buffer_remote, rate_limit_remote for stream source / sink
907 my @rsh_stream_buffer_in = $href->{stream_sink} ? stream_buffer_cmd_text($stream_options->{rsh_sink}) : ();
908 my @rsh_stream_buffer_out = $href->{stream_source} ? stream_buffer_cmd_text($stream_options->{rsh_source}) : ();
909
910 my @rsh_cmd_pipe = (
911 @decompress_in,
912 @rsh_stream_buffer_in,
913 $href,
914 @rsh_stream_buffer_out,
915 @rsh_compress_out,
916 );
917 @decompress_in = ();
918
919 # fixup redirect_to_file
920 if((scalar(@rsh_cmd_pipe) == 1) && ($rsh_cmd_pipe[0]->{redirect_to_file} || $rsh_cmd_pipe[0]->{append_to_file})) {
921 # NOTE: direct redirection in ssh command does not work: "ssh '> outfile'"
922 # we need to assemble: "ssh 'cat > outfile'"
923 unshift @rsh_cmd_pipe, { cmd_text => 'cat' };
924 }
925
926 my $rsh_text = _safe_cmd($href->{rsh}, \@unsafe_cmd);
927 return undef unless(defined($rsh_text));
928
929 $href->{cmd_text} = $rsh_text . ' ' . quoteshell(_piped_cmd_txt(\@rsh_cmd_pipe));
930 }
931
932 # local stream_buffer, rate_limit and show_progress in front of stream sink
933 my @stream_buffer_in = $href->{stream_sink} ? stream_buffer_cmd_text($stream_options->{local_sink}) : ();
934
935 push @cmd_pipe, (
936 @decompress_in, # empty if rsh
937 @stream_buffer_in,
938 @rsh_compress_in, # empty if not rsh
939 $href, # command or rsh_cmd_pipe
940 );
941 }
942
943 my $cmd = _piped_cmd_txt(\@cmd_pipe);
944
945 if(scalar(@unsafe_cmd)) {
946 ERROR "Unsafe command `$cmd`", map "Offending string: \"$_\"", @unsafe_cmd;
947 return undef;
948 }
949
950 if($dryrun && $destructive) {
951 DEBUG "### (dryrun) $cmd";
952 return [];
953 }
954 DEBUG "### $cmd";
955
956
957 # execute command
958 my ($pid, $out_fh, $err_fh, @stdout);
959 $err_fh = gensym;
960 if(eval_quiet { $pid = open3(undef, $out_fh, $err_fh, $cmd); }) {
961 chomp(@stdout = readline($out_fh));
962 chomp(@stderr = readline($err_fh));
963 waitpid($pid, 0);
964 if($do_trace) {
965 if($large_output) {
966 TRACE "Command output lines=" . scalar(@stdout) . " (large_output, not dumped)";
967 } else {
968 TRACE map("[stdout] $_", @stdout);
969 }
970 TRACE map("[stderr] $_", @stderr);
971 }
972 }
973 else {
974 ERROR "Command execution failed ($!): `$cmd`";
975 return undef;
976 }
977
978 # fatal errors
979 if($? == -1) {
980 ERROR "Command execution failed ($!): `$cmd`";
981 return undef;
982 }
983 elsif ($? & 127) {
984 my $signal = $? & 127;
985 ERROR "Command execution failed (child died with signal $signal): `$cmd`";
986 return undef;
987 }
988 my $exitcode = $? >> 8;
989
990 # call hooks: fatal_stderr, filter_stderr
991 if(($exitcode == 0) && $fatal_stderr) {
992 $exitcode = -1 if(grep &{$fatal_stderr}(), @stderr);
993 }
994 foreach my $filter_fn (@filter_stderr) {
995 @stderr = map { &{$filter_fn} ($exitcode); $_ // () } @stderr;
996 }
997
998 if($exitcode) {
999 unshift @stderr, "sh: $cmd";
1000 if($has_rsh && ($exitcode == 255)) {
1001 # SSH returns exit status 255 if an error occurred (including
1002 # network errors, dns failures).
1003 unshift @stderr, "(note: option \"ssh_identity\" is not set, using ssh defaults)" unless(grep /^-i$/, @$has_rsh);
1004 unshift @stderr, "SSH command failed (exitcode=$exitcode)";
1005 } else {
1006 unshift @stderr, "Command execution failed (exitcode=$exitcode)";
1007 }
1008 DEBUG @stderr;
1009 return undef;
1010 }
1011 else {
1012 DEBUG "Command execution successful";
1013 }
1014 return \@stdout;
1015 }
1016
1017
1018 sub _btrfs_filter_stderr
1019 {
1020 if(/^usage: / || /(unrecognized|invalid) option/) {
1021 WARN_ONCE "Using unsupported btrfs-progs < v$BTRFS_PROGS_MIN";
1022 }
1023 # strip error prefix (we print our own)
1024 # note that this also affects ssh_filter_btrbk.sh error strings
1025 s/^ERROR: //;
1026 }
1027
1028
1029 sub btrfs_filesystem_show($)
1030 {
1031 my $vol = shift || die;
1032 my $path = $vol->{PATH} // die;
1033 return run_cmd( cmd => vinfo_cmd($vol, "btrfs filesystem show", { unsafe => $path } ),
1034 rsh => vinfo_rsh($vol),
1035 non_destructive => 1,
1036 filter_stderr => \&_btrfs_filter_stderr,
1037 );
1038 }
1039
1040
1041 sub btrfs_filesystem_df($)
1042 {
1043 my $vol = shift || die;
1044 my $path = $vol->{PATH} // die;
1045 return run_cmd( cmd => vinfo_cmd($vol, "btrfs filesystem df", { unsafe => $path }),
1046 rsh => vinfo_rsh($vol),
1047 non_destructive => 1,
1048 filter_stderr => \&_btrfs_filter_stderr,
1049 );
1050 }
1051
1052
1053 sub btrfs_filesystem_usage($)
1054 {
1055 my $vol = shift || die;
1056 my $path = $vol->{PATH} // die;
1057 my $ret = run_cmd( cmd => vinfo_cmd($vol, "btrfs filesystem usage", { unsafe => $path } ),
1058 rsh => vinfo_rsh($vol),
1059 non_destructive => 1,
1060 filter_stderr => \&_btrfs_filter_stderr,
1061 );
1062 unless(defined($ret)) {
1063 ERROR "Failed to fetch btrfs filesystem usage for: $vol->{PRINT}", @stderr;
1064 return undef;
1065 }
1066
1067 return undef unless(defined($ret));
1068
1069 my %detail;
1070 foreach(@$ret) {
1071 $detail{device_size} = $1, next if(/^\s+Device size:\s+(\S+)/);
1072 $detail{device_allocated} = $1, next if(/^\s+Device allocated:\s+(\S+)/);
1073 $detail{device_unallocated} = $1, next if(/^\s+Device unallocated:\s+(\S+)/);
1074 $detail{device_missing} = $1, next if(/^\s+Device missing:\s+(\S+)/);
1075 $detail{device_used} = $1, next if(/^\s+Used:\s+(\S+)/);
1076 @detail{qw(free free_min)} = ($1,$2), next if(/^\s+Free \(estimated\):\s+(\S+)\s+\(min: (\S+)\)/);
1077 $detail{data_ratio} = $1, next if(/^\s+Data ratio:\s+([0-9]+\.[0-9]+)/);
1078 $detail{metadata_ratio} = $1, next if(/^\s+Metadata ratio:\s+([0-9]+\.[0-9]+)/);
1079 $detail{used} = $1, next if(/^\s+Used:\s+(\S+)/);
1080 @detail{qw(global_reserve global_reserve_used)} = ($1,$2), next if(/^\s+Global reserve:\s+(\S+)\s+\(used: (\S+)\)/);
1081 TRACE "Failed to parse filesystem usage line \"$_\" for: $vol->{PRINT}" if($do_trace);
1082 }
1083 DEBUG "Parsed " . scalar(keys %detail) . " filesystem usage detail items: $vol->{PRINT}";
1084
1085 foreach (qw(device_size device_used data_ratio)) {
1086 unless(defined($detail{$_})) {
1087 ERROR "Failed to parse filesystem usage detail (unsupported btrfs-progs) for: $vol->{PRINT}";
1088 return undef;
1089 }
1090 }
1091
1092 # calculate aggregate size / usage
1093 if($detail{device_size} =~ /^([0-9]+\.[0-9]+)(.*)/) {
1094 $detail{size} = sprintf('%.2f%s', $1 / $detail{data_ratio}, $2);
1095 }
1096 if($detail{device_used} =~ /^([0-9]+\.[0-9]+)(.*)/) {
1097 $detail{used} = sprintf('%.2f%s', $1 / $detail{data_ratio}, $2);
1098 }
1099
1100 TRACE(Data::Dumper->Dump([\%detail], ["btrfs_filesystem_usage($vol->{URL})"])) if($do_trace && $do_dumper);
1101 return \%detail;
1102 }
1103
1104
1105 # returns hashref with keys: (uuid parent_uuid id gen cgen top_level)
1106 # for btrfs root, returns at least: (id is_root)
1107 # for btrfs-progs >= 4.1, also returns key: "received_uuid"
1108 # if present, also returns (unvalidated) keys: (name creation_time flags)
1109 sub btrfs_subvolume_show($;@)
1110 {
1111 my $vol = shift || die;
1112 my %opts = @_;
1113 my @cmd_options;
1114 push(@cmd_options, '--rootid=' . $opts{rootid}) if($opts{rootid}); # btrfs-progs >= 4.12
1115 my $path = $vol->{PATH} // die;
1116 my $ret = run_cmd(cmd => vinfo_cmd($vol, "btrfs subvolume show", @cmd_options, { unsafe => $path }),
1117 rsh => vinfo_rsh($vol),
1118 non_destructive => 1,
1119 filter_stderr => \&_btrfs_filter_stderr,
1120 );
1121
1122 return undef unless(defined($ret));
1123
1124 unless(scalar(@$ret)) {
1125 ERROR "Failed to parse subvolume detail (unsupported btrfs-progs) for: $vol->{PRINT}";
1126 return undef;
1127 }
1128
1129 # NOTE: the first line starts with a path:
1130 # - btrfs-progs < 4.12 prints the full (absolute, resolved) path
1131 # - btrfs-progs >= 4.12 prints the relative path to btrfs root (or "/" if it is the root)
1132
1133 my %detail;
1134 if($ret->[0] =~ / is (btrfs root|toplevel subvolume)$/) {
1135 # btrfs-progs < 4.4 prints: "<subvol> is btrfs root"
1136 # btrfs-progs >= 4.4 prints: "<subvol> is toplevel subvolume"
1137 # btrfs-progs >= 4.8.3 does not enter here, as output shares format with regular subvolumes
1138 $detail{id} = 5;
1139 }
1140 else {
1141 my %trans = (
1142 "Name" => "name",
1143 "uuid" => "uuid",
1144 "UUID" => "uuid", # btrfs-progs >= 4.1
1145 "Parent uuid" => "parent_uuid",
1146 "Parent UUID" => "parent_uuid", # btrfs-progs >= 4.1
1147 "Received UUID" => "received_uuid", # btrfs-progs >= 4.1
1148 "Creation time" => "creation_time",
1149 "Object ID" => "id",
1150 "Subvolume ID" => "id", # btrfs-progs >= 4.1
1151 "Generation (Gen)" => "gen",
1152 "Generation" => "gen", # btrfs-progs >= 4.1
1153 "Gen at creation" => "cgen",
1154 "Parent" => "parent_id",
1155 "Parent ID" => "parent_id", # btrfs-progs >= 4.1
1156 "Top Level" => "top_level",
1157 "Top level ID" => "top_level", # btrfs-progs >= 4.1
1158 "Flags" => "flags",
1159 "Send transid" => "send_transid", # btrfs-progs >= 5.14.2
1160 "Send time" => "send_time", # btrfs-progs >= 5.14.2
1161 "Receive transid" => "receive_transid", # btrfs-progs >= 5.14.2
1162 "Receive time" => "receive_time", # btrfs-progs >= 5.14.2
1163 );
1164 foreach(@$ret) {
1165 next unless /^\s+(.+):\s+(.*)$/;
1166 my ($key, $value) = ($1, $2);
1167 if($trans{$key}) {
1168 $detail{$trans{$key}} = $value;
1169 } else {
1170 DEBUG "Ignoring subvolume detail \"$key: $value\" for: $vol->{PRINT}";
1171 }
1172 }
1173 DEBUG "Parsed " . scalar(keys %detail) . " subvolume detail items: $vol->{PRINT}";
1174
1175 # NOTE: as of btrfs-progs v4.6.1, flags are either "-" or "readonly"
1176 $detail{readonly} = ($detail{flags} =~ /readonly/) ? 1 : 0 if($detail{flags});
1177
1178 # validate required keys
1179 unless((defined($detail{parent_uuid}) && (($detail{parent_uuid} eq '-') || ($detail{parent_uuid} =~ /^$uuid_match$/))) &&
1180 (defined($detail{id}) && ($detail{id} =~ /^\d+$/) && ($detail{id} >= 5)) &&
1181 (defined($detail{gen}) && ($detail{gen} =~ /^\d+$/)) &&
1182 (defined($detail{cgen}) && ($detail{cgen} =~ /^\d+$/)) &&
1183 (defined($detail{top_level}) && ($detail{top_level} =~ /^\d+$/)) &&
1184 (defined($detail{readonly})))
1185 {
1186 ERROR "Failed to parse subvolume detail (unsupported btrfs-progs) for: $vol->{PRINT}";
1187 return undef;
1188 }
1189
1190 # NOTE: filesystems created with btrfs-progs < 4.16 have no UUID for subvolid=5,
1191 # assert {uuid} is either valid or undef
1192 if(defined($detail{uuid}) && ($detail{uuid} !~ /^$uuid_match$/)) {
1193 if($detail{id} == 5) {
1194 DEBUG "No UUID on btrfs root (id=5): $vol->{PRINT}";
1195 } else {
1196 ERROR "Failed to parse subvolume detail (unsupported btrfs-progs) for: $vol->{PRINT}";
1197 return undef;
1198 }
1199 delete $detail{uuid};
1200 }
1201
1202 # NOTE: received_uuid is not required here, as btrfs-progs < 4.1 does not give us that information.
1203 # no worries, we get this from btrfs_subvolume_list() for all subvols.
1204 if(defined($detail{received_uuid}) && ($detail{received_uuid} ne '-') && ($detail{received_uuid} !~ /^$uuid_match$/)) {
1205 ERROR "Failed to parse subvolume detail (unsupported btrfs-progs) for: $vol->{PRINT}";
1206 return undef;
1207 }
1208
1209 VINFO(\%detail, "detail") if($loglevel >=4);
1210 }
1211
1212 if($opts{rootid} && ($detail{id} != $opts{rootid})) {
1213 ERROR "Failed to parse subvolume detail (rootid mismatch) for: $vol->{PRINT}";
1214 return undef;
1215 }
1216
1217 if($detail{id} == 5) {
1218 DEBUG "Found btrfs root: $vol->{PRINT}";
1219 $detail{is_root} = 1;
1220 }
1221
1222 return \%detail;
1223 }
1224
1225
1226 sub btrfs_subvolume_list_readonly_flag($)
1227 {
1228 my $vol = shift || die;
1229 my $path = $vol->{PATH} // die;
1230
1231 my $ret = run_cmd(cmd => vinfo_cmd($vol, "btrfs subvolume list", '-a', '-r', { unsafe => $path } ),
1232 rsh => vinfo_rsh($vol),
1233 non_destructive => 1,
1234 filter_stderr => \&_btrfs_filter_stderr,
1235 );
1236 return undef unless(defined($ret));
1237
1238 my %ro;
1239 foreach(@$ret) {
1240 unless(/^ID\s+([0-9]+)\s+gen\s+[0-9]+\s+top level\s+[0-9]+\s+path\s/) {
1241 ERROR "Failed to parse subvolume list (unsupported btrfs-progs) for: $vol->{PRINT}";
1242 DEBUG "Offending line: $_";
1243 return undef;
1244 }
1245 $ro{$1} = 1;
1246 }
1247 DEBUG "Parsed " . scalar(keys %ro) . " readonly subvolumes for filesystem at: $vol->{PRINT}";
1248 return \%ro;
1249 }
1250
1251
1252 sub btrfs_subvolume_list($;@)
1253 {
1254 my $vol = shift || die;
1255 my %opts = @_;
1256 my $path = $vol->{PATH} // die;
1257 my @filter_options = ('-a');
1258 push(@filter_options, '-o') if($opts{subvol_only});
1259 push(@filter_options, '-d') if($opts{deleted_only});
1260
1261 # NOTE: btrfs-progs <= 3.17 do NOT support the '-R' flag.
1262 # NOTE: Support for btrfs-progs <= 3.17 has been dropped in
1263 # btrbk-0.23, the received_uuid flag very essential!
1264 my @display_options = ('-c', '-u', '-q', '-R');
1265 my $ret = run_cmd(cmd => vinfo_cmd($vol, "btrfs subvolume list", @filter_options, @display_options, { unsafe => $path } ),
1266 rsh => vinfo_rsh($vol),
1267 non_destructive => 1,
1268 filter_stderr => \&_btrfs_filter_stderr,
1269 );
1270 return undef unless(defined($ret));
1271
1272 my @nodes;
1273 foreach(@$ret)
1274 {
1275 my %node;
1276 # NOTE: btrfs-progs >= 4.13.2 pads uuid's with 36 whitespaces
1277 unless(/^ID \s+ ([0-9]+) \s+
1278 gen \s+ ([0-9]+) \s+
1279 cgen \s+ ([0-9]+) \s+
1280 top\ level \s+ ([0-9]+) \s+
1281 parent_uuid \s+ ([0-9a-f-]+) \s+
1282 received_uuid \s+ ([0-9a-f-]+) \s+
1283 uuid \s+ ([0-9a-f-]+) \s+
1284 path \s+ (.+) $/x) {
1285 ERROR "Failed to parse subvolume list (unsupported btrfs-progs) for: $vol->{PRINT}";
1286 DEBUG "Offending line: $_";
1287 return undef;
1288 }
1289 %node = (
1290 id => $1,
1291 gen => $2,
1292 cgen => $3,
1293 top_level => $4,
1294 parent_uuid => $5, # note: parent_uuid="-" if no parent
1295 received_uuid => $6,
1296 uuid => $7,
1297 path => $8 # btrfs path, NOT filesystem path
1298 );
1299
1300 # NOTE: "btrfs subvolume list <path>" prints <FS_TREE> prefix only if
1301 # the subvolume is reachable within <path>. (as of btrfs-progs-3.18.2)
1302 #
1303 # NOTE: Be prepared for this to change in btrfs-progs!
1304 $node{path} =~ s/^<FS_TREE>\///; # remove "<FS_TREE>/" portion from "path".
1305
1306 push @nodes, \%node;
1307 }
1308 DEBUG "Parsed " . scalar(@nodes) . " total subvolumes for filesystem at: $vol->{PRINT}";
1309
1310 return \@nodes;
1311 }
1312
1313
1314 sub btrfs_subvolume_list_complete($)
1315 {
1316 my $vol = shift || die;
1317
1318 # fetch subvolume list
1319 my $nodes = btrfs_subvolume_list($vol);
1320 return undef unless($nodes);
1321
1322 # fetch readonly flag
1323 # NOTE: the only way to get "readonly" flag is via a second call to "btrfs subvol list" with the "-r" option (as of btrfs-progs v4.3.1)
1324 my $ro = btrfs_subvolume_list_readonly_flag($vol);
1325 return undef unless(defined($ro));
1326 foreach (@$nodes) {
1327 $_->{readonly} = $ro->{$_->{id}} // 0;
1328 }
1329
1330 # btrfs root (id=5) is not provided by btrfs_subvolume_list above, read it separately (best-efford)
1331 my $tree_root = btrfs_subvolume_show($vol, rootid => 5);
1332 unless($tree_root) {
1333 # this is not an error:
1334 # - btrfs-progs < 4.12 does not support rootid lookup
1335 # - UUID can be missing if filesystem was created with btrfs-progs < 4.16
1336 DEBUG "Failed to fetch subvolume detail (old btrfs-progs?) for btrfs root (id=5) on: $vol->{PRINT}";
1337 $tree_root = { id => 5, is_root => 1 };
1338 }
1339 unshift(@$nodes, $tree_root);
1340
1341 return $nodes;
1342 }
1343
1344
1345 sub btrfs_subvolume_find_new($$;$)
1346 {
1347 my $vol = shift || die;
1348 my $path = $vol->{PATH} // die;
1349 my $lastgen = shift // die;
1350 my $ret = run_cmd(cmd => vinfo_cmd($vol, "btrfs subvolume find-new", { unsafe => $path }, $lastgen ),
1351 rsh => vinfo_rsh($vol),
1352 non_destructive => 1,
1353 filter_stderr => \&_btrfs_filter_stderr,
1354 large_output => 1,
1355 );
1356 unless(defined($ret)) {
1357 ERROR "Failed to fetch modified files for: $vol->{PRINT}", @stderr;
1358 return undef;
1359 }
1360
1361 my %files;
1362 my $parse_errors = 0;
1363 my $transid_marker;
1364 foreach(@$ret)
1365 {
1366 if(/^inode \S+ file offset (\S+) len (\S+) disk start \S+ offset \S+ gen (\S+) flags (\S+) (.+)$/) {
1367 my $file_offset = $1;
1368 my $len = $2;
1369 my $gen = $3;
1370 my $flags = $4;
1371 my $name = $5;
1372 $files{$name}->{len} += $len;
1373 $files{$name}->{new} = 1 if($file_offset == 0);
1374 $files{$name}->{gen}->{$gen} = 1; # count the generations
1375 if($flags ne "NONE") {
1376 $files{$name}->{flags}{$_} = 1 foreach split(/\|/, $flags);
1377 }
1378 }
1379 elsif(/^transid marker was (\S+)$/) {
1380 $transid_marker = $1;
1381 }
1382 else {
1383 ERROR "Failed to parse output from `btrfs subvolume find-new`:", $_;
1384 $parse_errors++;
1385 }
1386 }
1387 ERROR "Failed to parse $parse_errors lines from `btrfs subvolume find-new`" if($parse_errors);
1388
1389 return { files => \%files,
1390 transid_marker => $transid_marker,
1391 parse_errors => $parse_errors,
1392 };
1393 }
1394
1395
1396 # returns $target, or undef on error
1397 sub btrfs_subvolume_snapshot($$)
1398 {
1399 my $svol = shift || die;
1400 my $target_vol = shift // die;
1401 my $target_path = $target_vol->{PATH} // die;
1402 my $src_path = $svol->{PATH} // die;
1403 INFO "[snapshot] source: $svol->{PRINT}";
1404 INFO "[snapshot] target: $target_vol->{PRINT}";
1405 start_transaction("snapshot",
1406 vinfo_prefixed_keys("target", $target_vol),
1407 vinfo_prefixed_keys("source", $svol),
1408 );
1409 my $ret = run_cmd(cmd => vinfo_cmd($svol, "btrfs subvolume snapshot", '-r', { unsafe => $src_path }, { unsafe => $target_path } ),
1410 rsh => vinfo_rsh($svol),
1411 filter_stderr => \&_btrfs_filter_stderr,
1412 );
1413 end_transaction("snapshot", defined($ret));
1414 unless(defined($ret)) {
1415 ERROR "Failed to create snapshot: $svol->{PRINT} -> $target_path", @stderr;
1416 return undef;
1417 }
1418 return $target_vol;
1419 }
1420
1421
1422 sub btrfs_subvolume_delete($@)
1423 {
1424 my $vol = shift // die;
1425 my %opts = @_;
1426 my $target_type = $vol->{node}{TARGET_TYPE} || "";
1427 my $ret;
1428 INFO "[delete] target: $vol->{PRINT}";
1429 start_transaction($opts{type} // "delete", vinfo_prefixed_keys("target", $vol));
1430 if($target_type eq "raw") {
1431 $ret = run_cmd(cmd => [ 'rm', '-f',
1432 { unsafe => $vol->{PATH}, postfix => ($vol->{node}{BTRBK_RAW}{split} && ".split_??") },
1433 { unsafe => $vol->{PATH}, postfix => ".info" },
1434 ],
1435 rsh => vinfo_rsh($vol),
1436 );
1437 }
1438 else {
1439 my @options;
1440 push @options, "--commit-each" if($opts{commit});
1441 $ret = run_cmd(cmd => vinfo_cmd($vol, "btrfs subvolume delete", @options, { unsafe => $vol->{PATH} } ),
1442 rsh => vinfo_rsh($vol),
1443 fatal_stderr => sub { m/^ERROR: /; }, # probably not needed, "btrfs sub delete" returns correct exit status
1444 filter_stderr => \&_btrfs_filter_stderr,
1445 );
1446 }
1447 end_transaction($opts{type} // "delete", defined($ret));
1448
1449 unless(defined($ret)) {
1450 ERROR "Failed to delete subvolume: $vol->{PRINT}", @stderr;
1451 return undef;
1452 }
1453
1454 return $vol;
1455 }
1456
1457
1458 sub btrfs_qgroup_destroy($@)
1459 {
1460 my $vol = shift // die;
1461 my %opts = @_;
1462 my $vol_id = $vol->{node}{id};
1463 unless($vol_id) {
1464 ERROR "Unknown subvolume_id for: $vol->{PRINT}";
1465 return undef;
1466 }
1467 my $path = $vol->{PATH} // die;
1468 my $qgroup_id = "0/$vol_id";
1469 INFO "[qgroup-destroy] qgroup_id: $qgroup_id";
1470 INFO "[qgroup-destroy] subvolume: $vol->{PRINT}";
1471 start_transaction($opts{type} // "qgroup_destroy",
1472 vinfo_prefixed_keys("target", $vol));
1473 my $ret = run_cmd(cmd => vinfo_cmd($vol, "btrfs qgroup destroy", $qgroup_id, { unsafe => $path }),
1474 rsh => vinfo_rsh($vol),
1475 filter_stderr => \&_btrfs_filter_stderr,
1476 );
1477 end_transaction($opts{type} // "qgroup_destroy", defined($ret));
1478 unless(defined($ret)) {
1479 ERROR "Failed to destroy qgroup \"$qgroup_id\" for subvolume: $vol->{PRINT}", @stderr;
1480 return undef;
1481 }
1482 return $vol;
1483 }
1484
1485 sub _btrfs_send_options($$;$$)
1486 {
1487 my $snapshot = shift;
1488 my $target = shift;
1489 my $parent = shift;
1490 my $clone_src = shift // [];
1491 my $send_protocol = config_key($target, "send_protocol");
1492 my $send_compressed_data = config_key($target, "send_compressed_data");
1493 my @send_options;
1494 push(@send_options, '-p', { unsafe => $parent->{PATH} } ) if($parent);
1495 push(@send_options, '-c', { unsafe => $_ } ) foreach(map { $_->{PATH} } @$clone_src);
1496 push(@send_options, '--proto', $send_protocol ) if($send_protocol);
1497 push(@send_options, '--compressed-data' ) if($send_compressed_data);
1498 #push(@send_options, '-v') if($loglevel >= 3);
1499 return \@send_options;
1500 }
1501
1502 sub btrfs_send_receive($$;$$$)
1503 {
1504 my $snapshot = shift || die;
1505 my $target = shift || die;
1506 my $parent = shift;
1507 my $clone_src = shift // [];
1508 my $ret_vol_received = shift;
1509
1510 my $vol_received = vinfo_child($target, $snapshot->{NAME});
1511 $$ret_vol_received = $vol_received if(ref $ret_vol_received);
1512
1513 print STDOUT "Creating backup: $vol_received->{PRINT}\n" if($show_progress && (not $dryrun));
1514
1515 INFO "[send/receive] target: $vol_received->{PRINT}";
1516 INFO "[send/receive] source: $snapshot->{PRINT}";
1517 INFO "[send/receive] parent: $parent->{PRINT}" if($parent);
1518 INFO "[send/receive] clone-src: $_->{PRINT}" foreach(@$clone_src);
1519
1520 my $stream_options = config_stream_hash($snapshot, $target);
1521 my $compat_ignore_err = config_key_lru($target, "compat", "ignore_receive_errors");
1522
1523 my $send_options = _btrfs_send_options($snapshot, $target, $parent, $clone_src);
1524 my @receive_options;
1525 push(@receive_options, '--max-errors=0') if($compat_ignore_err);
1526
1527 my @cmd_pipe;
1528 push @cmd_pipe, {
1529 cmd => vinfo_cmd($snapshot, "btrfs send", @$send_options, { unsafe => $snapshot->{PATH} }),
1530 rsh => vinfo_rsh($snapshot, disable_compression => $stream_options->{stream_compress}),
1531 stream_options => $stream_options,
1532 filter_stderr => [ \&_btrfs_filter_stderr, sub { $_ = undef if(/^At subvol/) } ],
1533 };
1534
1535 push @cmd_pipe, {
1536 cmd => vinfo_cmd($target, "btrfs receive", @receive_options, { unsafe => $target->{PATH} . '/' } ),
1537 rsh => vinfo_rsh($target, disable_compression => $stream_options->{stream_compress}),
1538 fatal_stderr => sub {
1539 # NOTE: btrfs-progs < 4.11: if "btrfs send" fails, "btrfs receive" returns 0!
1540 if($compat_ignore_err && s/^ERROR: (.*)//) {
1541 WARN "Ignoring btrfs receive error (compat=ignore_receive_errors): $1";
1542 }
1543 m/^ERROR: /;
1544 },
1545 };
1546
1547 my $send_receive_error = 0;
1548 start_transaction("send-receive",
1549 vinfo_prefixed_keys("target", $vol_received),
1550 vinfo_prefixed_keys("source", $snapshot),
1551 vinfo_prefixed_keys("parent", $parent),
1552 );
1553 my $ret = run_cmd(@cmd_pipe);
1554 my @cmd_err;
1555 unless(defined($ret)) {
1556 @cmd_err = @stderr; # save for later
1557 $send_receive_error = 1;
1558 }
1559
1560 # Read in target subvolume metadata (btrfs subvolume show):
1561 # Double checking the output increases robustness against exotic
1562 # revisions of external commands (btrfs-progs, pv, xz, lz4, ...).
1563 #
1564 # NOTE: we cannot rely on the underlying shell to have
1565 # "pipefail" functionality.
1566 #
1567 # NOTE: btrfs-progs < 4.11:
1568 # "cat /dev/null | btrfs receive" returns with exitcode=0 and no
1569 # error message, having the effect that silently no subvolume is
1570 # created if any command in @cmd_pipe fail.
1571 my $is_garbled;
1572 if($dryrun) {
1573 INFO "[send/receive] (dryrun, skip) checking target metadata: $vol_received->{PRINT}";
1574 }
1575 else {
1576 INFO "[send/receive] checking target metadata: $vol_received->{PRINT}";
1577 my $detail = btrfs_subvolume_show($vol_received);
1578 if(defined($detail)) {
1579 unless($send_receive_error) {
1580 # plausibility checks on target detail
1581 unless($detail->{readonly}) {
1582 push @cmd_err, "target is not readonly: $vol_received->{PRINT}";
1583 $send_receive_error = 1;
1584 }
1585 if($detail->{received_uuid} && ($detail->{received_uuid} eq '-')) {
1586 # NOTE: received_uuid is not in @required_keys (needs btrfs-progs >= 4.1 (BTRFS_PROGS_MIN))
1587 # so we only check it if it's really present
1588 push @cmd_err, "received_uuid is not set on target: $vol_received->{PRINT}";
1589 $send_receive_error = 1;
1590 }
1591 if($parent && ($detail->{parent_uuid} eq '-')) {
1592 push @cmd_err, "parent_uuid is not set on target: $vol_received->{PRINT}";
1593 $send_receive_error = 1;
1594 }
1595 if((not $parent) && ($detail->{parent_uuid} ne '-')) {
1596 push @cmd_err, "parent_uuid is set on target: $vol_received->{PRINT}";
1597 $send_receive_error = 1;
1598 }
1599 }
1600
1601 # incomplete received (garbled) subvolumes are not readonly and have no received_uuid
1602 $is_garbled = ((not $detail->{readonly}) && defined($detail->{received_uuid}) && ($detail->{received_uuid} eq '-'));
1603 }
1604 else {
1605 push @cmd_err, "failed to check target subvolume: $vol_received->{PRINT}", @stderr;
1606 $send_receive_error = 1;
1607 }
1608 }
1609
1610 end_transaction("send-receive", not $send_receive_error);
1611
1612 if($send_receive_error) {
1613 ERROR "Failed to send/receive subvolume: $snapshot->{PRINT}" . ($parent ? " [$parent->{PATH}]" : "") . " -> $vol_received->{PRINT}", @cmd_err;
1614 }
1615
1616 if($is_garbled) {
1617 # NOTE: btrfs-progs does not delete incomplete received (garbled) subvolumes,
1618 # we need to do this by hand.
1619 # TODO: remove this as soon as btrfs-progs handle receive errors correctly.
1620 if(btrfs_subvolume_delete($vol_received, commit => "after", type => "delete_garbled")) {
1621 WARN "Deleted partially received (garbled) subvolume: $vol_received->{PRINT}";
1622 } else {
1623 WARN "Deletion of partially received (garbled) subvolume failed, assuming clean environment: $vol_received->{PRINT}";
1624 }
1625 }
1626 return $send_receive_error ? undef : 1;
1627 }
1628
1629
1630 sub btrfs_send_to_file($$$;$$)
1631 {
1632 my $source = shift || die;
1633 my $target = shift || die;
1634 my $parent = shift;
1635 my $ret_vol_received = shift;
1636 my $ret_raw_info = shift;
1637 my $target_path = $target->{PATH} // die;
1638 my $parent_uuid = $parent ? $parent->{node}{uuid} : undef ;
1639 my $received_uuid = $source->{node}{uuid};
1640 die unless($received_uuid);
1641 die if($parent && !$parent_uuid);
1642
1643 # prepare raw_info (for vinfo_inject_child)
1644 my %raw_info = (
1645 TYPE => 'raw',
1646 RECEIVED_UUID => $received_uuid,
1647 INCOMPLETE => 1,
1648 );
1649
1650 my $target_filename = $source->{NAME} || die;
1651 $target_filename .= ".btrfs";
1652
1653 my $compress = config_compress_hash($target, "raw_target_compress");
1654 my $encrypt = config_encrypt_hash($target, "raw_target_encrypt");
1655 my $split = config_key($target, "raw_target_split");
1656 my $stream_options = config_stream_hash($source, $target);
1657
1658 # make sure we dont re-compress, override "stream_compress" with "raw_target_compress"
1659 $stream_options->{stream_compress} = $compress if($compress);
1660
1661 my $send_options = _btrfs_send_options($source, $target, $parent);
1662 my @cmd_pipe;
1663 push @cmd_pipe, {
1664 cmd => vinfo_cmd($source, "btrfs send", @$send_options, { unsafe => $source->{PATH} } ),
1665 rsh => vinfo_rsh($source, disable_compression => $stream_options->{stream_compress}),
1666 stream_options => $stream_options,
1667 filter_stderr => [ \&_btrfs_filter_stderr, sub { $_ = undef if(/^At subvol/) } ],
1668 fatal_stderr => sub { m/^ERROR: /; },
1669 };
1670
1671 if($compress) {
1672 $raw_info{compress} = $compression{$compress->{key}}->{format};
1673 $target_filename .= '.' . $compression{$compress->{key}}->{format};
1674 push @cmd_pipe, { compress_stdin => $compress }; # does nothing if already compressed by stream_compress
1675 }
1676 if($encrypt) {
1677 $target_filename .= ($encrypt->{type} eq "gpg") ? '.gpg' : '.encrypted';
1678 }
1679
1680 # NOTE: $ret_vol_received must always be set when function returns!
1681 my $vol_received = vinfo_child($target, $target_filename);
1682 $$ret_vol_received = $vol_received if(ref $ret_vol_received);
1683
1684 if($encrypt) {
1685 $raw_info{encrypt} = $encrypt->{type};
1686
1687 if($encrypt->{type} eq "gpg") {
1688 # NOTE: We set "--no-random-seed-file" since one of the btrbk
1689 # design principles is to never create any files unasked. Enabling
1690 # "--no-random-seed-file" creates ~/.gnupg/random_seed, and as
1691 # such depends on $HOME to be set correctly (which e.g. is set to
1692 # "/" by some cron daemons). From gpg2(1) man page:
1693 # --no-random-seed-file GnuPG uses a file to store its
1694 # internal random pool over invocations This makes random
1695 # generation faster; however sometimes write operations are not
1696 # desired. This option can be used to achieve that with the cost
1697 # of slower random generation.
1698 my @gpg_options = ( '--batch', '--no-tty', '--no-random-seed-file', '--trust-model', 'always' );
1699 push @gpg_options, ( '--compress-algo', 'none' ) if($compress); # NOTE: if --compress-algo is not set, gpg might still compress according to OpenPGP standard.
1700 push(@gpg_options, ( '--no-default-keyring', '--keyring', { unsafe => $encrypt->{keyring} } )) if($encrypt->{keyring});
1701 if($encrypt->{recipient}) {
1702 push(@gpg_options, '--no-default-recipient');
1703 push(@gpg_options, map +( '--recipient', $_ ), @{$encrypt->{recipient}});
1704 }
1705 push @cmd_pipe, {
1706 cmd => [ 'gpg', @gpg_options, '--encrypt' ],
1707 compressed_ok => ($compress ? 1 : 0),
1708 };
1709 }
1710 elsif($encrypt->{type} eq "openssl_enc") {
1711 # encrypt using "openssl enc"
1712 $raw_info{cipher} = $encrypt->{ciphername};
1713
1714 # NOTE: iv is always generated locally!
1715 my $iv_size = $encrypt->{iv_size};
1716 my $iv;
1717 if($iv_size) {
1718 INFO "Generating iv for openssl encryption (cipher=$encrypt->{ciphername})";
1719 $iv = system_urandom($iv_size, 'hex');
1720 unless($iv) {
1721 ERROR "Failed generate IV for openssl_enc: $source->{PRINT}";
1722 return undef;
1723 }
1724 $raw_info{iv} = $iv;
1725 }
1726
1727 my $encrypt_key;
1728 if($encrypt->{keyfile}) {
1729 if($encrypt->{kdf_backend}) {
1730 WARN "Both openssl_keyfile and kdf_backend are configured, ignoring kdf_backend!";
1731 }
1732 $encrypt_key = '$(cat ' . quoteshell($encrypt->{keyfile}) . ')';
1733 }
1734 elsif($encrypt->{kdf_backend}) {
1735 if($encrypt->{kdf_keygen_each}) {
1736 $kdf_session_key = undef;
1737 %kdf_vars = ();
1738 }
1739 if($kdf_session_key) {
1740 INFO "Reusing session key for: $vol_received->{PRINT}";
1741 }
1742 else {
1743 # run kdf backend, set session key and vars
1744 DEBUG "Generating session key for: $vol_received->{PRINT}";
1745 my $key_target_text = $encrypt->{kdf_keygen_each} ? "\"$vol_received->{PRINT}\"" : "all raw backups";
1746
1747 print STDOUT "\nGenerate session key for $key_target_text:\n";
1748 my $kdf_values = run_cmd(cmd => [ { unsafe => $encrypt->{kdf_backend} }, $encrypt->{kdf_keysize} ],
1749 non_destructive => 1,
1750 );
1751 unless(defined($kdf_values)) {
1752 ERROR "Failed to generate session key for $key_target_text", @stderr;
1753 return undef;
1754 }
1755
1756 return undef unless(defined($kdf_values));
1757 foreach(@$kdf_values) {
1758 chomp;
1759 next if /^\s*$/; # ignore empty lines
1760 if(/^KEY=([0-9a-fA-f]+)/) {
1761 $kdf_session_key = $1;
1762 }
1763 elsif(/^([a-z_]+)=($raw_info_value_match)/) {
1764 my $info_key = 'kdf_' . $1;
1765 my $info_val = $2;
1766 DEBUG "Adding raw_info from kdf_backend: $info_key=$info_val";
1767 $kdf_vars{$info_key} = $info_val;
1768 }
1769 else {
1770 ERROR "Ambiguous line from kdf_backend: $encrypt->{kdf_backend}";
1771 return undef;
1772 }
1773 }
1774 unless($kdf_session_key && (length($kdf_session_key) == ($encrypt->{kdf_keysize} * 2))) {
1775 ERROR "Ambiguous key value from kdf_backend: $encrypt->{kdf_backend}";
1776 return undef;
1777 }
1778 INFO "Generated session key for: $vol_received->{PRINT}";
1779 }
1780 $encrypt_key = $kdf_session_key;
1781 %raw_info = ( %kdf_vars, %raw_info );
1782 }
1783
1784 my @openssl_options = (
1785 '-' . $encrypt->{ciphername},
1786 '-K', $encrypt_key,
1787 );
1788 push @openssl_options, ('-iv', $iv) if($iv);
1789
1790 push @cmd_pipe, {
1791 cmd => [ 'openssl', 'enc', '-e', @openssl_options ],
1792 compressed_ok => ($compress ? 1 : 0),
1793 };
1794 }
1795 else {
1796 die "Usupported encryption type (raw_target_encrypt)";
1797 }
1798 }
1799
1800 if($split) {
1801 # NOTE: we do not append a ".split" suffix on $target_filename here, as this propagates to ".info" file
1802 $raw_info{split} = $split;
1803 push @cmd_pipe, {
1804 cmd => [ 'split', '-b', uc($split), '-', { unsafe => "${target_path}/${target_filename}.split_" } ],
1805 rsh => vinfo_rsh($target, disable_compression => $stream_options->{stream_compress}),
1806 compressed_ok => ($compress ? 1 : 0),
1807 }
1808 }
1809 else {
1810 push @cmd_pipe, {
1811 # NOTE: We use "dd" instead of shell redirections here, as it is
1812 # common to have special filesystems (like NFS, SMB, FUSE) mounted
1813 # on $target_path. By using "dd" we make sure to write in
1814 # reasonably large blocks (default=128K), which is not always the
1815 # case when using redirections (e.g. "gpg > outfile" writes in 8K
1816 # blocks).
1817 # Another approach would be to always pipe through "cat", which
1818 # uses st_blksize from fstat(2) (with a minimum of 128K) to
1819 # determine the block size.
1820 cmd => [ 'dd', 'status=none', 'bs=' . config_key($target, "raw_target_block_size"), { prefix => "of=", unsafe => "${target_path}/${target_filename}" } ],
1821 #redirect_to_file => { unsafe => "${target_path}/${target_filename}" }, # alternative (use shell redirection), less overhead on local filesystems (barely measurable):
1822 rsh => vinfo_rsh($target, disable_compression => $stream_options->{stream_compress}),
1823 compressed_ok => ($compress ? 1 : 0),
1824 };
1825 }
1826
1827 $raw_info{FILE} = $target_filename;
1828 $raw_info{RECEIVED_PARENT_UUID} = $parent_uuid if($parent_uuid);
1829 # disabled for now, as its not very useful and might leak information:
1830 # $raw_info{parent_url} = $parent->{URL} if($parent);
1831 # $raw_info{target_url} = $vol_received->{URL};
1832 $$ret_raw_info = \%raw_info if($ret_raw_info);
1833
1834 print STDOUT "Creating raw backup: $vol_received->{PRINT}\n" if($show_progress && (not $dryrun));
1835
1836 INFO "[send-to-raw] target: $vol_received->{PRINT}";
1837 INFO "[send-to-raw] source: $source->{PRINT}";
1838 INFO "[send-to-raw] parent: $parent->{PRINT}" if($parent);
1839
1840 start_transaction("send-to-raw",
1841 vinfo_prefixed_keys("target", $vol_received),
1842 vinfo_prefixed_keys("source", $source),
1843 vinfo_prefixed_keys("parent", $parent),
1844 );
1845 my $ret;
1846 $ret = system_write_raw_info($vol_received, \%raw_info);
1847
1848 my @cmd_err;
1849 if(defined($ret)) {
1850 $ret = run_cmd(@cmd_pipe);
1851 @cmd_err = @stderr unless(defined($ret)); # save for later
1852 }
1853 else {
1854 push @cmd_err, "failed to write raw .info file: $vol_received->{PATH}.info", @stderr;
1855 }
1856
1857 if(defined($ret)) {
1858 # Test target file for "exists and size > 0" after writing, as we
1859 # can not rely on the exit status of the command pipe, and a shell
1860 # redirection as well as "dd" always creates the target file.
1861 # Note that "split" does not create empty files.
1862 my $test_postfix = ($split ? ".split_aa" : "");
1863 my $check_file = "${target_path}/${target_filename}${test_postfix}";
1864 DEBUG "Testing target data file (non-zero size): $check_file";
1865 $ret = run_cmd(cmd => [ 'test', '-s', { unsafe => $check_file } ],
1866 rsh => vinfo_rsh($target),
1867 );
1868 if(defined($ret)) {
1869 delete $raw_info{INCOMPLETE};
1870 $ret = system_write_raw_info($vol_received, { INCOMPLETE => 0 }, append => 1);
1871 } else {
1872 push @cmd_err, "failed to check target file (not present or zero length): $check_file";
1873 }
1874 }
1875 end_transaction("send-to-raw", defined($ret));
1876 unless(defined($ret)) {
1877 ERROR "Failed to send btrfs subvolume to raw file: $source->{PRINT}" . ($parent ? " [$parent->{PATH}]" : "") . " -> $vol_received->{PRINT}", @cmd_err;
1878 return undef;
1879 }
1880 return 1;
1881 }
1882
1883
1884 sub system_list_mountinfo($)
1885 {
1886 my $vol = shift // die;
1887 my $file = '/proc/self/mountinfo'; # NOTE: /proc/self/mounts is deprecated
1888 my $ret = run_cmd(cmd => [ 'cat', $file ],
1889 rsh => vinfo_rsh($vol),
1890 non_destructive => 1,
1891 );
1892 return undef unless(defined($ret));
1893 unless(@$ret) {
1894 ERROR "Failed to parse \"$vol->{URL_PREFIX}$file\": no output";
1895 return undef;
1896 }
1897
1898 my @mountinfo;
1899 foreach(@$ret)
1900 {
1901 # https://www.kernel.org/doc/Documentation/filesystems/proc.txt
1902 unless(/^(?<mount_id>[0-9]+) # mount ID: unique identifier of the mount (may be reused after umount)
1903 \s(?<parent_id>[0-9]+) # parent ID: ID of parent (or of self for the top of the mount tree)
1904 \s(?<st_dev>[0-9]+:[0-9]+) # major:minor: value of st_dev for files on filesystem
1905 \s(?<fs_root>\S+) # root: root of the mount within the filesystem
1906 \s(?<mount_point>\S+) # mount point: mount point relative to the process's root
1907 \s(?<mount_options>\S+) # mount options: per mount options
1908 (\s\S+)* # optional fields: zero or more fields of the form "tag[:value]"
1909 \s- # separator: marks the end of the optional fields
1910 \s(?<fs_type>\S+) # filesystem type: name of filesystem of the form "type[.subtype]"
1911 \s(?<mount_source>\S+) # mount source: filesystem specific information or "none"
1912 \s(?<super_options>\S+)$ # super options: per super block options
1913 /x)
1914 {
1915 ERROR "Failed to parse \"$vol->{URL_PREFIX}$file\"";
1916 DEBUG "Offending line: $_";
1917 return undef;
1918 }
1919 my %line = %+;
1920
1921 unless(defined(check_file($line{mount_point}, { absolute => 1 }))) {
1922 ERROR "Ambiguous mount point in \"$vol->{URL_PREFIX}$file\": $line{mount_point}";
1923 return undef;
1924 }
1925
1926 # merge super_options and mount_options to MNTOPS.
1927 my %mntops;
1928 foreach (split(',', delete($line{super_options})),
1929 split(',', delete($line{mount_options})))
1930 {
1931 if(/^(.+?)=(.+)$/) {
1932 $mntops{$1} = $2;
1933 } else {
1934 $mntops{$_} = 1;
1935 }
1936 }
1937 $mntops{rw} = 0 if($mntops{ro}); # e.g. mount_options="ro", super_options="rw"
1938
1939 # decode values (octal, e.g. "\040" = whitespace)
1940 s/\\([0-7]{3})/chr(oct($1))/eg foreach(values %line, values %mntops);
1941
1942 $line{MNTOPS} = \%mntops;
1943 push @mountinfo, \%line;
1944 }
1945 # TRACE(Data::Dumper->Dump([\@mountinfo], ["mountinfo"])) if($do_trace && $do_dumper);
1946 return \@mountinfo;
1947 }
1948
1949
1950 sub system_testdir($)
1951 {
1952 my $vol = shift // die;
1953 my $path = $vol->{PATH} // die;
1954 my $ret = run_cmd(cmd => vinfo_cmd($vol, "test", '-d', { unsafe => $path } ),
1955 rsh => vinfo_rsh($vol),
1956 non_destructive => 1,
1957 );
1958 return undef unless(defined($ret));
1959 DEBUG "Directory exists: $vol->{PRINT}";
1960 return 1;
1961 }
1962
1963
1964 sub system_realpath($)
1965 {
1966 my $vol = shift // die;
1967 my $path = $vol->{PATH} // die;
1968 my $compat = config_key_lru($vol, "compat", "busybox");
1969 my @options = ("-v"); # report error messages
1970 push @options, "-e" unless($compat); # all components must exist (not available in busybox!)
1971 push @options, "-f" if($compat); # all but the last component must exist.
1972 my $ret = run_cmd(cmd => vinfo_cmd($vol, "readlink", @options, { unsafe => $path } ),
1973 rsh => vinfo_rsh($vol),
1974 non_destructive => 1,
1975 );
1976 return undef unless(defined($ret));
1977
1978 my $realpath = scalar(@$ret) ? (check_file($ret->[0], { absolute => 1 }) // "") : "";
1979 unless($realpath) {
1980 ERROR "Failed to parse output of `realpath` for \"$vol->{PRINT}\": \"$ret->[0]\"";
1981 return undef;
1982 }
1983 DEBUG "Real path for \"$vol->{PRINT}\" is: $realpath";
1984 return undef if($compat && !system_testdir($vol));
1985 return $realpath;
1986 }
1987
1988
1989 sub system_mkdir($)
1990 {
1991 my $vol = shift // die;
1992 my $path = $vol->{PATH} // die;;
1993 INFO "Creating directory: $vol->{PRINT}/";
1994 start_transaction("mkdir", vinfo_prefixed_keys("target", $vol));
1995 my $ret = run_cmd(cmd => [ 'mkdir', '-p', { unsafe => $path } ],
1996 rsh => vinfo_rsh($vol),
1997 );
1998 end_transaction("mkdir", defined($ret));
1999 return undef unless(defined($ret));
2000 delete $realpath_cache{$vol->{URL}};
2001 return 1;
2002 }
2003
2004
2005 sub system_read_raw_info_dir($)
2006 {
2007 my $droot = shift // die;
2008 my $ret = run_cmd(
2009 # NOTE: we cannot simply "cat" all files here, as it will fail if no files found
2010 cmd => [ 'find', { unsafe => $droot->{PATH} },
2011 '-maxdepth', '1',
2012 '-type', 'f',
2013 '!', '-size', '0',
2014 '-name', '\*.btrfs.\*info', # match ".btrfs[.gz|.bz2|.xz|...][.gpg].info"
2015 '-print0',
2016 '-exec', 'cat \{\} \;',
2017 '-exec', 'printf "\0\0" \;',
2018 ],
2019 rsh => vinfo_rsh($droot),
2020 non_destructive => 1,
2021 );
2022 unless(defined($ret)) {
2023 ERROR("Failed to read *.btrfs.*.info files in: $droot->{PATH}");
2024 return undef;
2025 }
2026
2027 my @raw_targets;
2028 foreach my $info_text (split "\000\000", join "\n", @$ret) {
2029 unless($info_text =~ s/^(.*?)\000//s) {
2030 ERROR("Error while parsing command output for: $droot->{PATH}");
2031 return undef;
2032 }
2033 my $info_file = check_file($1, { absolute => 1 }, error_statement => 'for raw info file')
2034 // return undef;
2035 my $name = ($info_file =~ s/^.*\///r);
2036 $name =~ s/\.info$//;
2037
2038 my $raw_info = {
2039 INFO_FILE => $info_file,
2040 NAME => $name,
2041 };
2042 foreach my $line (split "\n", $info_text) {
2043 my ($key, $value) = ($line =~ /^([a-zA-Z_]+)=(.*)/);
2044 next unless $key;
2045 if($key eq "FILE") {
2046 WARN("Ignoring ambiguous \"FILE=$value\" from raw info file, using \"$name\": $info_file") if($value ne $name);
2047 next;
2048 }
2049 unless($value =~ /^$raw_info_value_match$/) {
2050 ERROR("Failed to parse \"$key=$value\" in raw info file: $info_file");
2051 return undef;
2052 }
2053 $raw_info->{$key} = $value;
2054 }
2055
2056 # input validation (we need to abort here, or the backups will be resumed)
2057 unless($raw_info->{TYPE} && ($raw_info->{TYPE} eq 'raw')) {
2058 ERROR("Unsupported \"type\" in raw info file: $info_file");
2059 return undef;
2060 }
2061 unless($raw_info->{RECEIVED_UUID} && ($raw_info->{RECEIVED_UUID} =~ /^$uuid_match$/)) {
2062 ERROR("Missing/Illegal \"received_uuid\" in raw info file: $info_file");
2063 return undef;
2064 }
2065 if(defined $raw_info->{RECEIVED_PARENT_UUID}) {
2066 unless(($raw_info->{RECEIVED_PARENT_UUID} eq '-') || ($raw_info->{RECEIVED_PARENT_UUID} =~ /^$uuid_match$/)) {
2067 ERROR("Illegal \"RECEIVED_PARENT_UUID\" in raw info file: $info_file");
2068 return undef;
2069 }
2070 }
2071 else {
2072 $raw_info->{RECEIVED_PARENT_UUID} = '-';
2073 }
2074
2075 push @raw_targets, $raw_info;
2076 }
2077
2078 DEBUG("Parsed " . @raw_targets . " raw info files in path: $droot->{PATH}");
2079 TRACE(Data::Dumper->Dump([\@raw_targets], ["system_read_raw_info_dir($droot->{URL})"])) if($do_trace && $do_dumper);
2080
2081 return \@raw_targets;
2082 }
2083
2084
2085 sub system_write_raw_info($$;@)
2086 {
2087 my $vol = shift // die;
2088 my $raw_info = shift // die;
2089 my %opts = @_;
2090 my $append = $opts{append};
2091 my $info_file = $vol->{PATH} . '.info';
2092
2093 # sort by %raw_info_sort, then by key
2094 my @line = $append ? () : ("#btrbk-v$VERSION", "# Do not edit this file");
2095 my @subst;
2096 push @line, '#t=' . time;
2097 foreach(sort { (($raw_info_sort{$a} // 99) <=> ($raw_info_sort{$b} // 99)) || ($a cmp $b) } keys %$raw_info) {
2098 push @line, ($_ . '=%s');
2099 push @subst, $raw_info->{$_};
2100 }
2101
2102 DEBUG "Writing (" . ($append ? "append:" . join(",", keys %$raw_info) : "create") . ") raw info file: $info_file";
2103 my $ret = run_cmd(
2104 { cmd => [ 'printf', quoteshell(join('\n', @line, "")), map quoteshell($_), @subst ] },
2105 { ($append ? "append_to_file" : "redirect_to_file") => { unsafe => $info_file },
2106 rsh => vinfo_rsh($vol),
2107 });
2108 return undef unless(defined($ret));
2109
2110 return $info_file;
2111 }
2112
2113
2114 sub system_urandom($;$) {
2115 my $size = shift;
2116 my $format = shift || 'hex';
2117 die unless(($size > 0) && ($size <= 256)); # sanity check
2118
2119 unless(open(URANDOM, '<', '/dev/urandom')) {
2120 ERROR "Failed to open /dev/urandom: $!";
2121 return undef;
2122 }
2123 binmode URANDOM;
2124 my $rand;
2125 my $rlen = read(URANDOM, $rand, $size);
2126 close(FILE);
2127 unless(defined($rand) && ($rlen == $size)) {
2128 ERROR "Failed to read from /dev/urandom: $!";
2129 return undef;
2130 }
2131
2132 if($format eq 'hex') {
2133 my $hex = unpack('H*', $rand);
2134 die unless(length($hex) == ($size * 2)); # paranoia check
2135 return $hex;
2136 }
2137 elsif($format eq 'bin') {
2138 return $rand;
2139 }
2140 die "unsupported format";
2141 }
2142
2143 sub read_extentmap_cache($)
2144 {
2145 my $vol = shift;
2146 my $cache_dir = config_key($vol, 'cache_dir');
2147 return undef unless($cache_dir);
2148 my $uuid = $vol->{node}{uuid} // die;
2149 foreach (@$cache_dir) {
2150 my $file = "$_/${uuid}.extentmap.bin";
2151 next unless (-f $file);
2152
2153 DEBUG "Reading extentmap cache: $file";
2154 if(open(my $fh, '<:raw', $file)) {
2155 my @range;
2156 my $buf;
2157 read($fh, $buf, 24 + 8 * 2); # read header
2158 my ($v, $gen, $time) = unpack('a24Q<Q<', $buf);
2159 unless(($v =~ /^btrbk_extentmap_v1/) && $gen && $time) {
2160 ERROR "Ambigous cache file: $file";
2161 next;
2162 }
2163 if($gen != $vol->{node}{gen}) {
2164 WARN "Subvolume generation has changed (cache=$gen, subvol=$vol->{node}{gen}), ignoring cache: $file";
2165 next;
2166 }
2167 while(read $fh, $buf, 8 * 2) { # read / unpack two words
2168 push @range, [ unpack('Q<Q<', $buf) ];
2169 #TRACE "read_extentmap_cache: range " . join("..", @{$range[-1]});
2170 };
2171 DEBUG "Read " . scalar(@range) . " regions (gen=$gen, timestamp='" . localtime($time) . "') from: $file";
2172 return \@range;
2173 } else {
2174 ERROR "Failed to open '$file': $!";
2175 }
2176 }
2177 return undef;
2178 }
2179
2180 sub write_extentmap_cache($)
2181 {
2182 my $vol = shift;
2183 my $extmap = $vol->{EXTENTMAP};
2184 my $cache_dir = config_key($vol, 'cache_dir');
2185 return undef unless($extmap && $cache_dir);
2186 my $uuid = $vol->{node}{uuid} // die;
2187 foreach (@$cache_dir) {
2188 unless(-d $_) {
2189 WARN_ONCE "Ignoring cache_dir (not a directory): $_";
2190 next;
2191 }
2192 my $file = "$_/${uuid}.extentmap.bin";
2193
2194 INFO "Writing extentmap cache: $file";
2195 if(open(my $fh, '>:raw', $file)) {
2196 # pack Q: unsigned quad (64bit, Documentation/filesystems/fiemap.txt)
2197 print $fh pack('a24Q<Q<', "btrbk_extentmap_v1", $vol->{node}{gen}, time);
2198 print $fh pack('Q<*', map(@{$_}, @$extmap));
2199 close($fh);
2200 } else {
2201 ERROR "Failed to create '$file': $!";
2202 }
2203 }
2204 }
2205
2206 # returns extents range (sorted array of [start,end], inclusive) from FIEMAP ioctl
2207 sub filefrag_extentmap($)
2208 {
2209 my $vol = shift || die;
2210 my $starttime = time;
2211
2212 INFO("Fetching extent map (filefrag): $vol->{PRINT}");
2213
2214 # NOTE: this returns exitstatus=0 if file is not found, or no files found
2215 my $ret = run_cmd(cmd => [ 'find', { unsafe => $vol->{PATH} }, '-xdev', '-type', 'f',
2216 '-exec', 'filefrag -b1 -v \{\} +' ],
2217 large_output => 1);
2218 unless(defined($ret)) {
2219 ERROR "Failed to fetch extent map: $vol->{PRINT}", @stderr;
2220 return undef;
2221 }
2222 WARN_ONCE "Configuration option \"ignore_extent_data_inline=no\" not available for filefrag (please install \"IO::AIO\" perl module)" unless(config_key($vol, "ignore_extent_data_inline"));
2223
2224 my @range; # array of [start,end]
2225 foreach (@$ret) {
2226 #my $file = $1 if(/^File size of (.*?) is/);
2227 if(/^\s*[0-9]+:\s*[0-9]+\.\.\s*[0-9]+:\s*([0-9]+)\.\.\s*([0-9]+):/) {
2228 # NOTE: filefrag (v1.45.5) returns wrong (?) physical_offset for
2229 # "inline" regions unless run with `-b1` (blocksize=1) option.
2230 #
2231 # For btrfs file systems it does not make much sense to consider
2232 # the "inline" extents anyways: these are stored in metadata
2233 # section and are not really part of the used disk space.
2234 #
2235 # # filefrag -v MYFILE
2236 # File size of MYFILE is 2307 (1 block of 4096 bytes)
2237 # ext: logical_offset: physical_offset: length: expected: flags:
2238 # 0: 0.. 4095: 0.. 4095: 4096: last,not_aligned,inline,eof
2239 # # filefrag -v -b1 MYFILE
2240 # File size of MYFILE is 2307 (4096 block of 1 bytes)
2241 # ext: logical_offset: physical_offset: length: expected: flags:
2242 # 0: 0.. 4095: 0.. 4095: 4096: last,not_aligned,inline,eof
2243 next if(/inline/);
2244 push @range, [ $1, $2 ];
2245 }
2246 }
2247 DEBUG("Parsed " . scalar(@range) . " regions in " . (time - $starttime) . "s for: $vol->{PRINT}");
2248 return extentmap_merge(\@range);
2249 }
2250
2251
2252 # returns extents range (sorted array of [start,end], inclusive) from FIEMAP ioctl
2253 sub aio_extentmap($)
2254 {
2255 my $vol = shift || die;
2256 my $starttime = time;
2257 my $ignore_inline = config_key($vol, "ignore_extent_data_inline");
2258
2259 INFO("Fetching extent map: $vol->{PRINT}");
2260
2261 # NOTE: this returns exitstatus=0 if file is not found, or no files found
2262 my $ret = run_cmd(cmd => [ 'find', { unsafe => $vol->{PATH} }, '-xdev', '-type', 'f' ],
2263 large_output => 1 );
2264 unless(defined($ret)) {
2265 ERROR "Failed to find files in: $vol->{PRINT}", @stderr;
2266 return undef;
2267 }
2268
2269 DEBUG("Reading ioctl FIEMAP of " . scalar(@$ret) . " files");
2270
2271 IO::AIO::max_outstanding(128); # < 1024 (max file descriptors)
2272 IO::AIO::max_poll_reqs(32);
2273
2274 my @range;
2275 my $count = 0;
2276 my $inline_count = 0;
2277 foreach my $file (@$ret) {
2278 IO::AIO::aio_open($file, IO::AIO::O_RDONLY(), 0, sub {
2279 # graceful abort on file open errors (check $count below)
2280 return unless($_[0]); # [ $fh ]
2281
2282 # note: aio_fiemap returns byte range (not blocks)
2283 # see: Documentation/filesystems/fiemap.rst
2284 IO::AIO::aio_fiemap($_[0], 0, undef, 0, undef, sub {
2285 $count++;
2286 foreach(@{$_[0]}) { # [ $logical, $physical, $length, $flags ]
2287 if($_->[3] & IO::AIO::FIEMAP_EXTENT_DATA_INLINE()) {
2288 $inline_count++;
2289 next if($ignore_inline);
2290 WARN_ONCE "Ambigous inline region [$_->[1] .. $_->[1] + $_->[2] - 1] for $file" if((($_->[1] != 0) || ($_->[2] != 4096)));
2291 }
2292 push @range, [ $_->[1], $_->[1] + $_->[2] - 1 ];
2293 }
2294 });
2295 });
2296 # poll, or the above eats up all our filedescriptors
2297 IO::AIO::poll_cb(); # takes "max_outstanding" and "max_poll_reqs" settings
2298 }
2299
2300 IO::AIO::flush();
2301
2302 WARN "Failed to open $count / " . scalar(@$ret) . " files" if($count != scalar(@$ret));
2303
2304 DEBUG("Parsed " . scalar(@range) . " regions (" . ($ignore_inline ? "ignored " : "") . "$inline_count \"inline\") for $count files in " . (time - $starttime) . "s for: $vol->{PRINT}");
2305 return extentmap_merge(\@range);
2306 }
2307
2308
2309 sub extentmap_total_blocks($)
2310 {
2311 my $extmap = shift;
2312 my $count = 0;
2313 foreach(@{$extmap->{rmap}}) {
2314 $count += ($_->[1] - $_->[0] + 1);
2315 }
2316 return $count;
2317 }
2318
2319
2320 sub extentmap_size($)
2321 {
2322 my $extmap = shift; # merged ranges
2323 return undef unless($extmap);
2324 my $size = 0;
2325 foreach(@$extmap) {
2326 $size += $_->[1] - $_->[0] + 1;
2327 }
2328 return $size;
2329 }
2330
2331
2332 sub extentmap_merge(@) {
2333 return undef unless(scalar(@_));
2334 my @range = sort { $a->[0] <=> $b->[0] } map @$_, @_;
2335 my @merged;
2336 my $start = -1;
2337 my $end = -2;
2338 foreach (@range) {
2339 if($_->[0] <= $end + 1) {
2340 # range overlaps the preceeding one, or is adjacent to it
2341 $end = $_->[1] if($_->[1] > $end);
2342 }
2343 else {
2344 push @merged, [ $start, $end ] if($start >= 0);
2345 $start = $_->[0];
2346 $end = $_->[1];
2347 }
2348 }
2349 push @merged, [ $start, $end ] if($start >= 0);
2350 DEBUG "extentmap: merged " . scalar(@range) . " regions into " . scalar(@merged) . " regions";
2351 return \@merged;
2352 }
2353
2354
2355 # ( A \ B ) : data in A that is not in B (relative complement of B in A)
2356 sub extentmap_diff($$)
2357 {
2358 my $l = shift // die; # A, sorted
2359 my $r = shift; # B, sorted
2360 return $l unless($r); # A \ 0 = A
2361
2362 my $i = 0;
2363 my $rn = scalar(@$r);
2364 my @diff;
2365
2366 foreach(@$l) {
2367 my $l_start = $_->[0];
2368 my $l_end = $_->[1];
2369 while(($i < $rn) && ($r->[$i][1] < $l_start)) { # r_end < l_start
2370 # advance r to next overlapping
2371 $i++;
2372 }
2373 while(($i < $rn) && ($r->[$i][0] <= $l_end)) { # r_start <= l_end
2374 # while overlapping, advance l_start
2375 my $r_start = $r->[$i][0];
2376 my $r_end = $r->[$i][1];
2377
2378 push @diff, [ $l_start, $r_start - 1 ] if($l_start < $r_start);
2379 $l_start = $r_end + 1;
2380 last if($l_start > $l_end);
2381 $i++;
2382 }
2383 push @diff, [ $l_start, $l_end ] if($l_start <= $l_end);
2384 }
2385 DEBUG "extentmap: relative complement ( B=" . scalar(@$r) . ' \ A=' . scalar(@$l) . " ) = " . scalar(@diff) . " regions";
2386 return \@diff;
2387 }
2388
2389
2390 sub btr_tree($$$$)
2391 {
2392 my $vol = shift;
2393 my $vol_root_id = shift || die;
2394 my $mount_source = shift || die; # aka device
2395 my $mountpoints = shift || die; # all known mountpoints for this filesystem: arrayref of mountinfo
2396 die unless($vol_root_id >= 5);
2397
2398 # return parsed tree from %mount_source_cache if present
2399 my $host_mount_source = $vol->{URL_PREFIX} . $mount_source;
2400 my $cached_tree = $mount_source_cache{$host_mount_source};
2401 TRACE "mount_source_cache " . ($cached_tree ? "HIT" : "MISS") . ": $host_mount_source" if($do_trace);
2402 if($cached_tree) {
2403 TRACE "btr_tree: returning cached tree at id=$vol_root_id" if($do_trace);
2404 my $node = $cached_tree->{ID_HASH}{$vol_root_id};
2405 ERROR "Unknown subvolid=$vol_root_id in btrfs tree of $host_mount_source" unless($node);
2406 return $node;
2407 }
2408
2409 my $node_list = btrfs_subvolume_list_complete($vol);
2410 return undef unless(ref($node_list) eq "ARRAY");
2411 my $vol_root;
2412
2413 TRACE "btr_tree: processing subvolume list of: $vol->{PRINT}" if($do_trace);
2414
2415 # return a reference to the cached root if we already know the tree,
2416 # making sure every tree is only stored once, which is essential
2417 # e.g. when injecting nodes. die if duplicate UUID exist on
2418 # different file systems (no matter if local or remote).
2419 #
2420 # note: this relies on subvolume UUID's to be "universally unique"
2421 # (which is why cloning btrfs filesystems using "dd" is a bad idea)
2422 #
2423 # note: a better way would be to always compare the UUID of
2424 # subvolid=5. unfortunately this is not possible for filesystems
2425 # created with btrfs-progs < 4.16 (no UUID for subvolid=5).
2426 foreach(@$node_list) {
2427 my $node_uuid = $_->{uuid};
2428 next unless($node_uuid);
2429 if($uuid_cache{$node_uuid}) {
2430 # at least one uuid of $node_list is already known
2431 TRACE "uuid_cache HIT: $node_uuid" if($do_trace);
2432 $vol_root = $uuid_cache{$node_uuid}->{TREE_ROOT}->{ID_HASH}->{$vol_root_id};
2433 unless($vol_root) {
2434 # check for deleted subvolumes: e.g. still mounted, but deleted elsewhere
2435 my $deleted_nodes = btrfs_subvolume_list($vol, deleted_only => 1);
2436 return undef unless(ref($deleted_nodes) eq "ARRAY");
2437 if(grep ($_->{id} eq $vol_root_id), @$deleted_nodes) {
2438 ERROR "Subvolume is deleted: id=$vol_root_id mounted on: $vol->{PRINT}";
2439 return undef;
2440 }
2441 ERROR "Subvolume id=$vol_root_id is not present on known btrfs tree: $vol->{PRINT}",
2442 "Possible causes:",
2443 " - Mismatch in mountinfo",
2444 " - Subvolume was deleted while btrbk is running",
2445 " - Duplicate UUID present on multiple filesystems: $node_uuid";
2446 ERROR "Refusing to run on unstable environment; exiting";
2447 exit 1;
2448 }
2449 INFO "Assuming same filesystem: \"$vol_root->{TREE_ROOT}->{host_mount_source}\", \"$host_mount_source\"";
2450 TRACE "btr_tree: returning already parsed tree at id=$vol_root->{id}" if($do_trace);
2451 $mount_source_cache{$host_mount_source} = $vol_root->{TREE_ROOT};
2452 return $vol_root;
2453 }
2454 last; # check only first UUID (for performance)
2455 }
2456
2457 # fill our hashes and uuid_cache
2458 my %id;
2459 my %uuid_hash;
2460 my %received_uuid_hash;
2461 my %parent_uuid_hash;
2462 my $gen_max = 0;
2463 foreach my $node (@$node_list) {
2464 my $node_id = $node->{id};
2465 my $node_uuid = $node->{uuid};
2466 die unless($node_id >= 5);
2467 die "duplicate node id" if(exists($id{$node_id}));
2468 $id{$node_id} = $node;
2469 if($node_uuid) {
2470 # NOTE: uuid on btrfs root (id=5) is not always present
2471 $uuid_hash{$node_uuid} = $node;
2472 $uuid_cache{$node_uuid} = $node;
2473 # hacky: if root node has no "uuid", it also has no "received_uuid" and no "gen"
2474 push(@{$received_uuid_hash{$node->{received_uuid}}}, $node) if($node->{received_uuid} ne '-');
2475 push(@{$parent_uuid_hash{$node->{parent_uuid}}}, $node) if($node->{parent_uuid} ne '-');
2476 $gen_max = $node->{gen} if($node->{gen} > $gen_max);
2477 }
2478 elsif(not $node->{is_root}) {
2479 die "missing uuid on subvolume";
2480 }
2481 $node->{SUBTREE} = [];
2482 }
2483 my $tree_root = $id{5} // die "missing btrfs root";
2484 $tree_root->{ID_HASH} = \%id;
2485 $tree_root->{UUID_HASH} = \%uuid_hash;
2486 $tree_root->{RECEIVED_UUID_HASH} = \%received_uuid_hash;
2487 $tree_root->{PARENT_UUID_HASH} = \%parent_uuid_hash;
2488 $tree_root->{GEN_MAX} = $gen_max;
2489 $tree_root->{URL_PREFIX} = $vol->{URL_PREFIX}; # hacky, first url prefix for logging
2490
2491 # NOTE: host_mount_source is NOT dependent on MACHINE_ID:
2492 # if we return already present tree (see above), the value of
2493 # host_mount_source will still point to the mount_source of the
2494 # first machine.
2495 $tree_root->{mount_source} = $mount_source;
2496 $tree_root->{host_mount_source} = $host_mount_source; # unique identifier, e.g. "/dev/sda1" or "ssh://hostname[:port]/dev/sda1"
2497
2498 $vol_root = $id{$vol_root_id};
2499 unless($vol_root) {
2500 ERROR "Failed to resolve tree root for subvolid=$vol_root_id: " . ($vol->{PRINT} // $vol->{id});
2501 return undef;
2502 }
2503
2504 # set REL_PATH and tree references (TREE_ROOT, SUBTREE, TOP_LEVEL)
2505 foreach my $node (@$node_list) {
2506 unless($node->{is_root}) {
2507 # note: it is possible that id < top_level, e.g. after restoring
2508 my $top_level = $id{$node->{top_level}};
2509 die "missing top_level reference" unless(defined($top_level));
2510
2511 push(@{$top_level->{SUBTREE}}, $node);
2512 $node->{TOP_LEVEL} = $top_level;
2513
2514 # "path" always starts with set REL_PATH
2515 my $rel_path = $node->{path};
2516 unless($top_level->{is_root}) {
2517 die unless($rel_path =~ s/^\Q$top_level->{path}\E\///);
2518 }
2519 $node->{REL_PATH} = $rel_path; # relative to {TOP_LEVEL}->{path}
2520 }
2521 $node->{TREE_ROOT} = $tree_root;
2522 add_btrbk_filename_info($node);
2523 }
2524
2525 # add known mountpoints to nodes
2526 my %mountpoints_hash;
2527 foreach(@$mountpoints) {
2528 my $node_id = $_->{MNTOPS}{subvolid};
2529 my $node = $id{$node_id};
2530 unless($node) {
2531 WARN "Unknown subvolid=$node_id (in btrfs tree of $host_mount_source) for mountpoint: $vol->{URL_PREFIX}$_->{mount_point}";
2532 next;
2533 }
2534 $mountpoints_hash{$node_id} = $node;
2535 push @{$node->{MOUNTINFO}}, $_; # if present, node is mounted at MOUNTINFO
2536 }
2537 $tree_root->{MOUNTED_NODES} = [ (values %mountpoints_hash) ]; # list of mounted nodes
2538
2539 TRACE "btr_tree: returning tree at id=$vol_root->{id}" if($do_trace);
2540 VINFO($vol_root, "node") if($loglevel >=4);
2541
2542 $mount_source_cache{$host_mount_source} = $tree_root;
2543 return $vol_root;
2544 }
2545
2546
2547 sub btr_tree_inject_node($$$)
2548 {
2549 my $top_node = shift;
2550 my $detail = shift;
2551 my $rel_path = shift;
2552 my $subtree = $top_node->{SUBTREE} // die;
2553 my $tree_root = $top_node->{TREE_ROOT};
2554
2555 die unless($detail->{parent_uuid} && $detail->{received_uuid} && exists($detail->{readonly}));
2556
2557 $tree_inject_id -= 1;
2558 $tree_root->{GEN_MAX} += 1;
2559
2560 my $uuid = sprintf("${fake_uuid_prefix}%012u", -($tree_inject_id));
2561 my $node = {
2562 %$detail, # make a copy
2563 TREE_ROOT => $tree_root,
2564 SUBTREE => [],
2565 TOP_LEVEL => $top_node,
2566 REL_PATH => $rel_path,
2567 INJECTED => 1,
2568 id => $tree_inject_id,
2569 uuid => $uuid,
2570 gen => $tree_root->{GEN_MAX},
2571 cgen => $tree_root->{GEN_MAX},
2572 };
2573 push(@$subtree, $node);
2574 $uuid_cache{$uuid} = $node;
2575 $tree_root->{ID_HASH}->{$tree_inject_id} = $node;
2576 $tree_root->{UUID_HASH}->{$uuid} = $node;
2577 push( @{$tree_root->{RECEIVED_UUID_HASH}->{$node->{received_uuid}}}, $node ) if($node->{received_uuid} ne '-');
2578 push( @{$tree_root->{PARENT_UUID_HASH}->{$node->{parent_uuid}}}, $node ) if($node->{parent_uuid} ne '-');
2579 return $node;
2580 }
2581
2582
2583 # returns array of { path, mountinfo }
2584 # NOTE: includes subvolumes hidden by other mountpoint
2585 sub __fs_info
2586 {
2587 my $node = shift;
2588 my $url_prefix = shift;
2589 my @ret = $node->{MOUNTINFO} ? map +{ path => $url_prefix . $_->{mount_point}, mountinfo => $_ }, @{$node->{MOUNTINFO}} : ();
2590 return @ret if($node->{is_root});
2591 return ((map +{ path => $_->{path} . '/' . $node->{REL_PATH}, mountinfo => $_->{mountinfo} }, __fs_info($node->{TOP_LEVEL}, $url_prefix)), @ret);
2592 }
2593
2594 sub _fs_info
2595 {
2596 my $node = shift // die;
2597 my $url_prefix = shift // $node->{TREE_ROOT}{URL_PREFIX};
2598 my @ret = __fs_info($node, $url_prefix);
2599 @ret = ({ path => "$url_prefix<$node->{TREE_ROOT}{mount_source}>/$node->{path}",
2600 mountinfo => undef }) unless(scalar(@ret));
2601 return @ret;
2602 }
2603
2604 sub _fs_path
2605 {
2606 my @ret = map $_->{path}, _fs_info(@_);
2607 return wantarray ? @ret : $ret[0];
2608 }
2609
2610
2611 sub _is_correlated($$)
2612 {
2613 my $a = shift; # node a
2614 my $b = shift; # node b
2615 return 0 if($a->{is_root} || $b->{is_root});
2616 return 0 unless($a->{readonly} && $b->{readonly});
2617 return (($a->{uuid} eq $b->{received_uuid}) ||
2618 ($b->{uuid} eq $a->{received_uuid}) ||
2619 (($a->{received_uuid} ne '-') && ($a->{received_uuid} eq $b->{received_uuid})));
2620 }
2621
2622
2623 sub _is_same_fs_tree($$)
2624 {
2625 return ($_[0]->{TREE_ROOT}{host_mount_source} eq $_[1]->{TREE_ROOT}{host_mount_source});
2626 }
2627
2628
2629 sub _is_child_of
2630 {
2631 my $node = shift;
2632 my $uuid = shift;
2633 foreach(@{$node->{SUBTREE}}) {
2634 return 1 if($_->{uuid} eq $uuid);
2635 return 1 if(_is_child_of($_, $uuid));
2636 }
2637 return 0;
2638 }
2639
2640
2641 sub _get_longest_match
2642 {
2643 my $node = shift;
2644 my $path = shift;
2645 my $check_path = shift; # MUST have a trailing slash
2646 $path .= '/' unless($path =~ /\/$/); # correctly handle root path="/"
2647
2648 return undef unless($check_path =~ /^\Q$path\E/);
2649 foreach(@{$node->{SUBTREE}}) {
2650 my $ret = _get_longest_match($_, $path . $_->{REL_PATH}, $check_path);
2651 return $ret if($ret);
2652 }
2653 return { node => $node,
2654 path => $path };
2655 }
2656
2657
2658 sub vinfo($$)
2659 {
2660 my $url = shift // die;
2661 my $config = shift;
2662
2663 my ($url_prefix, $path) = check_url($url);
2664 die "invalid url: $url" unless(defined($path));
2665 my $print = $path;
2666 my $name = $path;
2667 $name =~ s/^.*\///;
2668 $name = '/' if($name eq "");
2669
2670 my $host = undef;
2671 my $port = undef;
2672 if($url_prefix) {
2673 $host = $url_prefix;
2674 die unless($host =~ s/^ssh:\/\///);
2675 $port = $1 if($host =~ s/:([1-9][0-9]*)$//);
2676 $print = $host . (defined($port) ? "[$port]:" : ":") . $path;
2677 $host =~ s/^\[//; # remove brackets from ipv6_addr
2678 $host =~ s/\]$//; # remove brackets from ipv6_addr
2679 }
2680
2681 # Note that PATH and URL have no trailing slash, except if "/".
2682 # Note that URL and URL_PREFIX can contain ipv6 address in brackets (e.g. "[::1]").
2683 return {
2684 HOST => $host, # hostname|ipv4_address|ipv6_address|<undef>
2685 PORT => $port, # port|<undef>
2686 NAME => $name,
2687 PATH => $path,
2688 PRINT => $print, # "hostname:/path" or "hostname[port]:/path"
2689 URL => $url_prefix . $path, # ssh://hostname[:port]/path
2690 URL_PREFIX => $url_prefix, # ssh://hostname[:port] (or "" if local)
2691 MACHINE_ID => $url_prefix || "LOCAL:", # unique: "LOCAL:" or hostname and port
2692 CONFIG => $config,
2693
2694 # These are added in vinfo_init_root
2695 #NODE_SUBDIR => undef,
2696 #VINFO_MOUNTPOINT => undef,
2697 }
2698 }
2699
2700
2701 sub vinfo_child($$;$)
2702 {
2703 my $parent = shift || die;
2704 my $rel_path = shift // die;
2705 my $config = shift; # override parent config
2706 my $name = $rel_path;
2707 my $subvol_dir = "";
2708 $subvol_dir = $1 if($name =~ s/^(.*)\///);
2709
2710 # Note that PATH and URL intentionally contain "//" if $parent->{PATH} = "/".
2711 my $vinfo = {
2712 HOST => $parent->{HOST},
2713 PORT => $parent->{PORT},
2714 NAME => $name,
2715 PATH => "$parent->{PATH}/$rel_path",
2716 PRINT => "$parent->{PRINT}" . ($parent->{PRINT} =~ /\/$/ ? "" : "/") . $rel_path,
2717 URL => "$parent->{URL}/$rel_path",
2718 URL_PREFIX => $parent->{URL_PREFIX},
2719 MACHINE_ID => $parent->{MACHINE_ID},
2720 CONFIG => $config // $parent->{CONFIG},
2721 VINFO_MOUNTPOINT => $parent->{VINFO_MOUNTPOINT},
2722
2723 # NOTE: these are NOT present in non-child vinfo, and should be used
2724 # only for printing and comparing results of vinfo_subvol_list.
2725 SUBVOL_PATH => $rel_path,
2726 SUBVOL_DIR => $subvol_dir, # SUBVOL_PATH=SUBVOL_DIR/NAME
2727 };
2728
2729 # TRACE "vinfo_child: created from \"$parent->{PRINT}\": $info{PRINT}" if($do_trace);
2730 return $vinfo;
2731 }
2732
2733
2734 sub vinfo_rsh($;@)
2735 {
2736 my $vinfo = shift || die;
2737 my %opts = @_;
2738 my $host = $vinfo->{HOST};
2739 return undef unless(defined($host));
2740
2741 my $config = $vinfo->{CONFIG};
2742 die unless($config);
2743
2744 # as of btrbk-0.28.0, ssh port is a property of a "vinfo", set with
2745 # "ssh://hostname[:port]" in 'volume' and 'target' sections. Note
2746 # that the port number is also used for the MACHINE_ID to
2747 # distinguish virtual machines on same host with different ports.
2748 my $ssh_port = $vinfo->{PORT};
2749 unless($ssh_port) {
2750 # PORT defaults to ssh_port (DEPRECATED)
2751 $ssh_port = config_key($config, "ssh_port") // "default";
2752 $ssh_port = undef if($ssh_port eq "default");
2753 }
2754 my $ssh_user = config_key($config, "ssh_user");
2755 my $ssh_identity = config_key($config, "ssh_identity");
2756 my $ssh_compression = config_key($config, "ssh_compression");
2757 my $ssh_cipher_spec = join(",", @{config_key($config, "ssh_cipher_spec")});
2758 my @ssh_options; # as of btrbk-0.29.0, we run ssh without -q (catching @stderr)
2759 push(@ssh_options, '-p', $ssh_port) if($ssh_port);
2760 push(@ssh_options, '-c', $ssh_cipher_spec) if($ssh_cipher_spec ne "default");
2761 push(@ssh_options, '-i', { unsafe => $ssh_identity }) if($ssh_identity); # NOTE: hackily used in run_cmd on errors
2762 if($opts{disable_compression}) {
2763 push(@ssh_options, '-o', 'compression=no'); # force ssh compression=no (in case it is defined in ssh_config)
2764 } elsif($ssh_compression) {
2765 push(@ssh_options, '-C');
2766 }
2767 my $ssh_dest = $ssh_user ? $ssh_user . '@' . $host : $host;
2768 return ['ssh', @ssh_options, $ssh_dest ];
2769 }
2770
2771
2772 sub vinfo_cmd($$@)
2773 {
2774 my $vinfo = shift || die;
2775 my $cmd = shift || die;
2776 my @cmd_args = @_;
2777 my $backend = config_key_lru($vinfo, "backend") // die;
2778 my $cmd_mapped = $backend_cmd_map{$backend}{$cmd} // [ split(" ", $cmd) ];
2779 return [ @$cmd_mapped, @cmd_args ];
2780 }
2781
2782 sub _get_btrbk_date(@)
2783 {
2784 my %a = @_; # named capture buffers (%+) from $btrbk_timestamp_match
2785
2786 my @tm = ( ($a{ss} // 0), ($a{mm} // 0), ($a{hh} // 0), $a{DD}, ($a{MM} - 1), ($a{YYYY} - 1900) );
2787 my $NN = $a{NN} // 0;
2788 my $zz = $a{zz};
2789 my $has_exact_time = defined($a{hh}); # false if timestamp_format=short
2790
2791 my $time;
2792 if(defined($zz)) {
2793 eval_quiet { $time = timegm(@tm); };
2794 } else {
2795 eval_quiet { $time = timelocal(@tm); };
2796 }
2797 unless(defined($time)) {
2798 # WARN "$@"; # sadly Time::Local croaks, which also prints the line number from here.
2799 return undef;
2800 }
2801
2802 # handle ISO 8601 time offset
2803 if(defined($zz)) {
2804 my $offset;
2805 if($zz eq 'Z') {
2806 $offset = 0; # Zulu time == UTC
2807 }
2808 elsif($zz =~ /^([+-])([0-9][0-9])([0-9][0-9])$/) {
2809 $offset = ( $3 * 60 ) + ( $2 * 60 * 60 );
2810 $offset *= -1 if($1 eq '-');
2811 }
2812 else {
2813 return undef;
2814 }
2815 $time -= $offset;
2816 }
2817
2818 return [ $time, $NN, $has_exact_time ];
2819 }
2820
2821 sub add_btrbk_filename_info($;$)
2822 {
2823 my $node = shift;
2824 my $raw_info = shift;
2825 my $name = $node->{REL_PATH};
2826 return undef unless(defined($name));
2827
2828 # NOTE: unless long-iso file format is encountered, the timestamp is interpreted in local timezone.
2829
2830 $name =~ s/^(.*)\///;
2831 if($raw_info && ($name =~ /^(?<name>.+)\.$btrbk_timestamp_match$raw_postfix_match$/)) { ; }
2832 elsif((not $raw_info) && ($name =~ /^(?<name>.+)\.$btrbk_timestamp_match$/)) { ; }
2833 else {
2834 return undef;
2835 }
2836 $name = $+{name} // die;
2837 my $btrbk_date = _get_btrbk_date(%+); # use named capture buffers of previous match
2838 unless($btrbk_date) {
2839 WARN "Illegal timestamp on subvolume \"$node->{REL_PATH}\", ignoring";
2840 return undef;
2841 }
2842
2843 $node->{BTRBK_BASENAME} = $name;
2844 $node->{BTRBK_DATE} = $btrbk_date;
2845 $node->{BTRBK_RAW} = $raw_info if($raw_info);
2846 return $node;
2847 }
2848
2849
2850 sub _find_mountpoint($$)
2851 {
2852 my $root = shift;
2853 my $path = shift;
2854 $path .= '/' unless($path =~ /\/$/); # append trailing slash
2855 while (my $tree = $root->{SUBTREE}) {
2856 my $m = undef;
2857 foreach (@$tree) {
2858 $m = $_, last if($path =~ /^\Q$_->{mount_point}\E\//);
2859 }
2860 last unless defined $m;
2861 $root = $m;
2862 }
2863 TRACE "resolved mount point for \"$path\": $root->{mount_point} (mount_source=$root->{mount_source}, subvolid=" . ($root->{MNTOPS}->{subvolid} // '<undef>') . ")" if($do_trace);
2864 return $root;
2865 }
2866
2867 sub mountinfo_tree($)
2868 {
2869 my $vol = shift;
2870 my $mountinfo = $mountinfo_cache{$vol->{MACHINE_ID}};
2871 TRACE "mountinfo_cache " . ($mountinfo ? "HIT" : "MISS") . ": $vol->{MACHINE_ID}" if($do_trace);
2872 unless($mountinfo) {
2873 $mountinfo = system_list_mountinfo($vol);
2874 return undef unless($mountinfo);
2875 $mountinfo_cache{$vol->{MACHINE_ID}} = $mountinfo;
2876 }
2877 return $mountinfo->[0]->{TREE_ROOT} if($mountinfo->[0]->{TREE_ROOT});
2878
2879 my %id = map +( $_->{mount_id} => $_ ), @$mountinfo;
2880 my $tree_root;
2881 foreach my $node (@$mountinfo) {
2882 my $parent = $id{$node->{parent_id}};
2883 if($parent && ($node->{mount_id} != $node->{parent_id})) {
2884 $node->{PARENT} = $parent;
2885 push @{$parent->{SUBTREE}}, $node;
2886 } else {
2887 die "multiple root mount points" if($tree_root);
2888 $tree_root = $node;
2889 }
2890 # populate cache (mount points are always real paths)
2891 $realpath_cache{$vol->{URL_PREFIX} . $node->{mount_point}} = $node->{mount_point};
2892 }
2893 die "no root mount point" unless($tree_root);
2894 $_->{TREE_ROOT} = $tree_root foreach (@$mountinfo);
2895 $tree_root->{MOUNTINFO_LIST} = $mountinfo;
2896
2897 return $tree_root;
2898 }
2899
2900
2901 sub vinfo_mountpoint
2902 {
2903 my $vol = shift // die;
2904 my %args = @_;
2905
2906 DEBUG "Resolving mount point for: $vol->{PRINT}";
2907 my $mountinfo_root = mountinfo_tree($vol);
2908 return undef unless($mountinfo_root);
2909
2910 my $realpath = $realpath_cache{$vol->{URL}};
2911 unless(defined($realpath)) {
2912 $realpath = system_realpath($vol);
2913 # set to empty string on errors (try only once)
2914 $realpath_cache{$vol->{URL}} = $realpath // "";
2915 }
2916 return undef unless($realpath);
2917
2918 my $mountpoint = _find_mountpoint($mountinfo_root, $realpath);
2919
2920 # handle autofs
2921 if($mountpoint->{fs_type} eq 'autofs') {
2922 if($args{autofs_retry}) {
2923 DEBUG "Non-btrfs autofs mount point for: $vol->{PRINT}";
2924 return undef;
2925 }
2926 DEBUG "Found autofs mount point, triggering automount on $mountpoint->{mount_point} for: $vol->{PRINT}";
2927 btrfs_subvolume_show(vinfo($vol->{URL_PREFIX} . $mountpoint->{mount_point}, $vol->{CONFIG}));
2928 $mountinfo_cache{$vol->{MACHINE_ID}} = undef;
2929 return vinfo_mountpoint($vol, %args, autofs_retry => 1);
2930 }
2931
2932 if($args{fs_type} && ($mountpoint->{fs_type} ne $args{fs_type})) {
2933 ERROR "Not a btrfs filesystem (mountpoint=\"$mountpoint->{mount_point}\", fs_type=\"$mountpoint->{fs_type}\"): $vol->{PRINT}";
2934 return undef;
2935 }
2936 DEBUG "Mount point for \"$vol->{PRINT}\": $mountpoint->{mount_point} (mount_source=$mountpoint->{mount_source}, fs_type=$mountpoint->{fs_type})";
2937 return ($realpath, $mountpoint);
2938 }
2939
2940
2941 sub vinfo_init_root($)
2942 {
2943 my $vol = shift || die;
2944
2945 @stderr = (); # clear @stderr (propagated for logging)
2946 my ($real_path, $mountpoint) = vinfo_mountpoint($vol, fs_type => 'btrfs');
2947 return undef unless($mountpoint);
2948
2949 my @same_source_mounts = grep { $_->{mount_source} eq $mountpoint->{mount_source} } @{$mountpoint->{TREE_ROOT}{MOUNTINFO_LIST}};
2950 foreach my $mnt (grep { !defined($_->{MNTOPS}{subvolid}) } @same_source_mounts) {
2951 # kernel <= 4.2 does not have subvolid=NN in /proc/self/mounts, read it with btrfs-progs
2952 DEBUG "No subvolid provided in mounts for: $mnt->{mount_point}";
2953 my $detail = btrfs_subvolume_show(vinfo($vol->{URL_PREFIX} . $mnt->{mount_point}, $vol->{CONFIG}));
2954 return undef unless($detail);
2955 $mnt->{MNTOPS}{subvolid} = $detail->{id} || die; # also affects %mountinfo_cache
2956 }
2957
2958 # read btrfs tree for the mount point
2959 @stderr = (); # clear @stderr (propagated for logging)
2960 my $mnt_path = $mountpoint->{mount_point};
2961 my $mnt_vol = vinfo($vol->{URL_PREFIX} . $mnt_path, $vol->{CONFIG});
2962 my $mnt_tree_root = btr_tree($mnt_vol, $mountpoint->{MNTOPS}{subvolid}, $mountpoint->{mount_source}, \@same_source_mounts);
2963 return undef unless($mnt_tree_root);
2964
2965 # find longest match in btrfs tree
2966 $real_path .= '/' unless($real_path =~ /\/$/); # correctly handle root path="/"
2967 my $ret = _get_longest_match($mnt_tree_root, $mnt_path, $real_path) // die;
2968 my $tree_root = $ret->{node};
2969 return undef unless($tree_root);
2970
2971 # set NODE_SUBDIR if $vol->{PATH} points to a regular (non-subvolume) directory.
2972 # in other words, "PATH=<path_to_subvolume>/NODE_SUBDIR"
2973 my $node_subdir = $real_path;
2974 die unless($node_subdir =~ s/^\Q$ret->{path}\E//); # NOTE: $ret->{path} has trailing slash!
2975 $node_subdir =~ s/\/+$//;
2976 $vol->{NODE_SUBDIR} = $node_subdir if($node_subdir ne '');
2977 $vol->{node} = $tree_root;
2978
2979 $vol->{VINFO_MOUNTPOINT} = vinfo($vol->{URL_PREFIX} . $mnt_path, $vol->{CONFIG});
2980 $vol->{VINFO_MOUNTPOINT}{node} = $mnt_tree_root;
2981
2982 return $tree_root;
2983 }
2984
2985
2986 sub vinfo_init_raw_root($;@)
2987 {
2988 my $droot = shift || die;
2989 my $tree_root = $raw_url_cache{$droot->{URL}};
2990 TRACE "raw_url_cache " . ($tree_root ? "HIT" : "MISS") . ": URL=$droot->{URL}" if($do_trace);
2991 unless($tree_root) {
2992 if(my $real_path = $realpath_cache{$droot->{URL}}) {
2993 my $real_url = $droot->{URL_PREFIX} . $real_path;
2994 $tree_root = $raw_url_cache{$real_url};
2995 TRACE "raw_url_cache " . ($tree_root ? "HIT" : "MISS") . ": REAL_URL=$real_url" if($do_trace);
2996 }
2997 }
2998
2999 unless($tree_root) {
3000 DEBUG "Creating raw subvolume list: $droot->{PRINT}";
3001
3002 # create fake btr_tree
3003 $tree_root = { id => 5,
3004 is_root => 1,
3005 mount_source => '@raw_tree', # for _fs_path (logging)
3006 host_mount_source => $droot->{URL} . '@raw_tree', # for completeness (this is never used)
3007 GEN_MAX => 1,
3008 SUBTREE => [],
3009 UUID_HASH => {},
3010 RECEIVED_UUID_HASH => {},
3011 URL_PREFIX => $droot->{URL_PREFIX}, # for _fs_path (logging)
3012 MOUNTINFO => [ { mount_point => $droot->{PATH} } ], # for _fs_path (logging)
3013 };
3014 $tree_root->{TREE_ROOT} = $tree_root;
3015
3016 # list and parse *.info
3017 my $raw_info_ary = system_read_raw_info_dir($droot);
3018 return undef unless($raw_info_ary);
3019
3020 # inject nodes to fake btr_tree
3021 $droot->{node} = $tree_root;
3022 my %child_uuid_list;
3023 foreach my $raw_info (@$raw_info_ary)
3024 {
3025 # Set btrfs subvolume information (received_uuid, parent_uuid) from filename info.
3026 #
3027 # NOTE: received_parent_uuid in BTRBK_RAW is the "parent of the source subvolume", NOT the
3028 # "parent of the received subvolume".
3029 my $subvol = vinfo_child($droot, $raw_info->{NAME});
3030 unless(vinfo_inject_child($droot, $subvol, {
3031 TARGET_TYPE => $raw_info->{TYPE},
3032 parent_uuid => '-', # NOTE: correct value gets inserted below
3033 # Incomplete raw fakes get same semantics as real subvolumes (readonly=0, received_uuid='-')
3034 received_uuid => ($raw_info->{INCOMPLETE} ? '-' : $raw_info->{RECEIVED_UUID}),
3035 readonly => ($raw_info->{INCOMPLETE} ? 0 : 1),
3036 }, $raw_info))
3037 {
3038 ERROR("Failed create raw node \"$raw_info->{NAME}\" from raw info file: \"$raw_info->{INFO_FILE}\"");
3039 return undef;
3040 }
3041
3042 if($raw_info->{RECEIVED_PARENT_UUID} ne '-') {
3043 $child_uuid_list{$raw_info->{RECEIVED_PARENT_UUID}} //= [];
3044 push @{$child_uuid_list{$raw_info->{RECEIVED_PARENT_UUID}}}, $subvol;
3045 }
3046 }
3047
3048 my @subvol_list = @{vinfo_subvol_list($droot, sort => 'path')};
3049 DEBUG "Found " . scalar(@subvol_list) . " raw subvolume backups in: $droot->{PRINT}";
3050
3051 foreach my $subvol (@subvol_list)
3052 {
3053 # If restoring a backup from raw btrfs images (using "incremental yes|strict"):
3054 # "btrfs send -p parent source > svol.btrfs", the backups
3055 # on the target will get corrupted (unusable!) as soon as
3056 # an any files in the chain gets deleted.
3057 #
3058 # We need to make sure btrbk will NEVER delete those:
3059 # - svol.<timestamp>--<received_uuid_0>.btrfs : root (full) image
3060 # - svol.<timestamp>--<received_uuid-n>[@<received_uuid_n-1>].btrfs : incremental image
3061
3062 foreach my $child (@{$child_uuid_list{$subvol->{node}{received_uuid}}}) {
3063 # Insert correct (i.e. fake) parent UUID
3064 $child->{node}{parent_uuid} = $subvol->{node}{uuid};
3065
3066 # Make sure that incremental backup chains are never broken:
3067 DEBUG "Found parent/child partners, forcing preserve of: \"$subvol->{PRINT}\", \"$child->{PRINT}\"";
3068 $subvol->{node}{FORCE_PRESERVE} = "preserve forced: parent of another raw target";
3069 $child->{node}{FORCE_PRESERVE} ||= "preserve forced: child of another raw target";
3070 }
3071 }
3072 # TRACE(Data::Dumper->Dump([\@subvol_list], ["vinfo_raw_subvol_list{$droot}"]));
3073 }
3074
3075 $droot->{node} = $tree_root;
3076 $droot->{VINFO_MOUNTPOINT} = $droot; # fake mountpoint
3077 $raw_url_cache{$droot->{URL}} = $tree_root;
3078
3079 return $tree_root;
3080 }
3081
3082
3083 sub _vinfo_subtree_list
3084 {
3085 my $tree = shift;
3086 my $vinfo_parent = shift;
3087 my $filter_readonly = shift; # if set, return only read-only
3088 my $filter_btrbk_direct_leaf = shift; # if set, return only read-only direct leafs matching btrbk_basename
3089 my $list = shift // [];
3090 my $path_prefix = shift // "";
3091 my $depth = shift // 0;
3092
3093 # if $vinfo_parent->{NODE_SUBDIR} is set, vinfo_parent->{PATH} does
3094 # not point to a subvolume directly, but to "<path_to_subvolume>/NODE_SUBDIR".
3095 # skip nodes wich are not in NODE_SUBDIR, or strip NODE_SUBDIR from from rel_path.
3096 my $node_subdir_filter = ($depth == 0) ? $vinfo_parent->{NODE_SUBDIR} : undef;
3097 foreach my $node (@{$tree->{SUBTREE}}) {
3098 my $rel_path = $node->{REL_PATH};
3099 if(defined($node_subdir_filter)) {
3100 next unless($rel_path =~ s/^\Q$node_subdir_filter\E\///);
3101 }
3102 my $path = $path_prefix . $rel_path; # always points to a subvolume
3103
3104 # filter direct leafs (SUBVOL_DIR="") matching btrbk_basename
3105 next unless(!defined($filter_btrbk_direct_leaf) ||
3106 (exists($node->{BTRBK_BASENAME}) && ($node->{BTRBK_BASENAME} eq $filter_btrbk_direct_leaf) &&
3107 ($rel_path !~ /\//))); # note: depth is always 0 if $filter_btrbk_direct_leaf
3108
3109 # filter readonly, push vinfo_child
3110 if(!$filter_readonly || $node->{readonly}) {
3111 my $vinfo = vinfo_child($vinfo_parent, $path);
3112 $vinfo->{node} = $node;
3113
3114 # add some additional information to vinfo
3115 $vinfo->{subtree_depth} = $depth;
3116
3117 push(@$list, $vinfo);
3118 }
3119
3120 unless(defined($filter_btrbk_direct_leaf)) {
3121 _vinfo_subtree_list($node, $vinfo_parent, $filter_readonly, undef, $list, $path . '/', $depth + 1);
3122 }
3123 }
3124 return $list;
3125 }
3126
3127
3128 sub vinfo_subvol_list($;@)
3129 {
3130 my $vol = shift || die;
3131 my %opts = @_;
3132
3133 TRACE "Creating subvolume list for: $vol->{PRINT}" if($do_trace);
3134
3135 # recurse into tree from $vol->{node}, returns arrayref of vinfo
3136 my $subvol_list = _vinfo_subtree_list($vol->{node}, $vol, $opts{readonly}, $opts{btrbk_direct_leaf});
3137
3138 if($opts{sort}) {
3139 if($opts{sort} eq 'path') {
3140 my @sorted = sort { $a->{SUBVOL_PATH} cmp $b->{SUBVOL_PATH} } @$subvol_list;
3141 return \@sorted;
3142 }
3143 else { die; }
3144 }
3145 return $subvol_list;
3146 }
3147
3148
3149 # returns vinfo_child if $node is in tree below $vol (or equal if allow_equal), or undef
3150 sub vinfo_resolved($$;@)
3151 {
3152 my $node = shift || die;
3153 my $vol = shift || die; # root vinfo node
3154 my %opts = @_;
3155 my $top_id = $vol->{node}{id};
3156 my @path;
3157 my $nn = $node;
3158 while(($nn->{id} != $top_id) && (!$nn->{is_root})) {
3159 unshift(@path, $nn->{REL_PATH});
3160 $nn = $nn->{TOP_LEVEL};
3161 }
3162 if(scalar(@path) == 0) {
3163 return $vol if($opts{allow_equal} && not defined($vol->{NODE_SUBDIR}));
3164 return undef;
3165 }
3166 return undef if($nn->{is_root} && (!$vol->{node}{is_root}));
3167 my $jpath = join('/', @path);
3168 if(defined($vol->{NODE_SUBDIR})) {
3169 return undef unless($jpath =~ s/^\Q$vol->{NODE_SUBDIR}\E\///);
3170 }
3171 if(defined($opts{btrbk_direct_leaf})) {
3172 return undef if($jpath =~ /\//);
3173 return undef unless(exists($node->{BTRBK_BASENAME}) && ($node->{BTRBK_BASENAME} eq $opts{btrbk_direct_leaf}))
3174 }
3175 my $vinfo = vinfo_child($vol, $jpath);
3176 $vinfo->{node} = $node;
3177 return $vinfo;
3178 }
3179
3180
3181 # returns vinfo if $node is below any mountpoint of $vol
3182 sub vinfo_resolved_all_mountpoints($$)
3183 {
3184 my $node = shift || die;
3185 my $vol = shift || die;
3186 my $tree_root = $vol->{node}{TREE_ROOT};
3187 foreach my $mnt_node (@{$tree_root->{MOUNTED_NODES}}) {
3188 foreach my $mountinfo (@{$mnt_node->{MOUNTINFO}}) {
3189 my $mnt_vol = vinfo($vol->{URL_PREFIX} . $mountinfo->{mount_point}, $vol->{CONFIG});
3190 $mnt_vol->{node} = $mnt_node;
3191 TRACE "vinfo_resolved_all_mountpoints: trying mountpoint: $mnt_vol->{PRINT}" if($do_trace);
3192 my $vinfo = vinfo_resolved($node, $mnt_vol, allow_equal => 1);
3193 return $vinfo if($vinfo);
3194 }
3195 }
3196 return undef;
3197 }
3198
3199 sub vinfo_subvol($$)
3200 {
3201 my $vol = shift || die;
3202 my $subvol_path = shift // die;
3203 foreach (@{vinfo_subvol_list($vol)}) {
3204 return $_ if($_->{SUBVOL_PATH} eq $subvol_path);
3205 }
3206 return undef;
3207 }
3208
3209
3210 sub vinfo_inject_child($$$;$)
3211 {
3212 my $vinfo = shift;
3213 my $vinfo_child = shift;
3214 my $detail = shift;
3215 my $raw_info = shift;
3216 my $node;
3217
3218 my $node_subdir = defined($vinfo->{NODE_SUBDIR}) ? $vinfo->{NODE_SUBDIR} . '/' : "";
3219 my $rel_path = $node_subdir . $vinfo_child->{SUBVOL_PATH};
3220
3221 $node = btr_tree_inject_node($vinfo->{node}, $detail, $rel_path);
3222 return undef unless(add_btrbk_filename_info($node, $raw_info));
3223
3224 $vinfo_child->{node} = $node;
3225 TRACE "vinfo_inject_child: injected child id=$node->{id} to $vinfo->{PRINT}" if($do_trace);
3226 return $vinfo_child;
3227 }
3228
3229
3230 # returns hash: ( $prefix_{url,path,host,name,subvol_path,rsh} => value, ... )
3231 sub vinfo_prefixed_keys($$)
3232 {
3233 my $prefix = shift // die;
3234 my $vinfo = shift;
3235 return () unless($vinfo);
3236 my %ret;
3237 if($prefix) {
3238 $ret{$prefix} = $vinfo->{PRINT};
3239 $prefix .= '_';
3240 }
3241 foreach (qw( URL PATH HOST PORT NAME )) {
3242 $ret{$prefix . lc($_)} = $vinfo->{$_};
3243 }
3244 $ret{$prefix . "subvolume"} = $vinfo->{PATH};
3245 my $rsh = vinfo_rsh($vinfo);
3246 $ret{$prefix . "rsh"} = $rsh ? _safe_cmd($rsh) : undef,
3247 return %ret;
3248 }
3249
3250
3251 sub vinfo_assign_config($;$)
3252 {
3253 my $vinfo = shift || die;
3254 my $vinfo_snapshot_root = shift;
3255 my $config = $vinfo->{CONFIG} || die;
3256 die if($config->{VINFO});
3257 $config->{VINFO} = $vinfo;
3258 $config->{VINFO_SNAPROOT} = $vinfo_snapshot_root;
3259 }
3260
3261
3262 sub vinfo_snapshot_root($)
3263 {
3264 my $vinfo = shift;
3265 return $vinfo->{CONFIG}{VINFO_SNAPROOT};
3266 }
3267
3268
3269 sub config_subsection($$;$)
3270 {
3271 my $config = shift || die;
3272 my $context = shift || die;
3273 die if grep($_->{CONTEXT} ne $context, @{$config->{SUBSECTION}});
3274 return @{$config->{SUBSECTION}};
3275 }
3276
3277 sub vinfo_subsection($$;$)
3278 {
3279 # if config: must have SUBSECTION key
3280 # if vinfo: must have CONFIG key
3281 my $config_or_vinfo = shift || die;
3282 my $context = shift || die;
3283 my $include_aborted = shift;
3284 my @config_list;
3285 my $vinfo_check;
3286 if(exists($config_or_vinfo->{SUBSECTION})) {
3287 @config_list = config_subsection($config_or_vinfo, $context);
3288 }
3289 else {
3290 @config_list = config_subsection($config_or_vinfo->{CONFIG}, $context);
3291 die unless($config_or_vinfo->{CONFIG}->{VINFO} == $config_or_vinfo); # check back reference
3292 }
3293
3294 return map {
3295 die unless($_->{VINFO} == $_->{VINFO}->{CONFIG}->{VINFO}); # check all back references
3296 ($include_aborted || !$_->{ABORTED}) ? $_->{VINFO} : ()
3297 } @config_list;
3298 }
3299
3300
3301 # allow (absolute) path / url with wildcards
3302 # allow group (exact match)
3303 # allow host[:port] (exact match)
3304 sub vinfo_filter_statement($) {
3305 my $filter = shift;
3306 my %ret = ( unparsed => $filter );
3307
3308 my ($url_prefix, $path) = check_url($filter, accept_wildcards => 1);
3309 unless($path) {
3310 # allow relative path with wildcards
3311 $url_prefix = "";
3312 $path = check_file($filter, { relative => 1, wildcards => 1 }, sanitize => 1);
3313 }
3314 if($path) {
3315 # support "*some*file*", "*/*"
3316 my $regex = join('[^\/]*', map(quotemeta($_), split(/\*+/, lc($url_prefix) . $path, -1)));
3317 if($path =~ /^\//) {
3318 $ret{url_regex} = qr/^$regex$/; # absolute path, match full string
3319 } else {
3320 $ret{url_regex} = qr/\/$regex$/; # match end of string
3321 }
3322 }
3323
3324 $ret{group_eq} = $filter if($filter =~ /^$group_match$/);
3325
3326 if($filter =~ /^(?<host>$host_name_match|$ipv4_addr_match|\[$ipv6_addr_match\])(:(?<port>[1-9][0-9]*))?$/) {
3327 my ($host, $port) = ( $+{host}, $+{port} );
3328 $host =~ s/^\[//; # remove brackets from ipv6_addr
3329 $host =~ s/\]$//; # remove brackets from ipv6_addr
3330 $ret{host_port_eq} = { host => $host, port => $port };
3331 }
3332 elsif($filter =~ /^$ipv6_addr_match$/) {
3333 $ret{host_port_eq} = { host => $filter } ;
3334 }
3335
3336 TRACE 'vinfo_filter_statement: filter="' . $filter . '" url_regex="' . ($ret{url_regex} // "<undef>") . '" group_eq="' . ($ret{group_eq} // "<undef>") . '" host_port_eq="' . ($ret{host_port_eq} ? $ret{host_port_eq}{host} . ":" . ($ret{host_port_eq}{port} // "<undef>") : "<undef>") . '"' if($do_trace);
3337 return undef unless(exists($ret{url_regex}) || exists($ret{group_eq}) || exists($ret{host_port_eq}));
3338 return \%ret;
3339 }
3340
3341
3342 sub vinfo_match($$;@)
3343 {
3344 my $filter = shift;
3345 my $vinfo = shift;
3346 my %opts = @_;
3347 my $flag_matched = $opts{flag_matched};
3348
3349 # never match dummy volume section
3350 return 0 if($vinfo->{CONFIG}{DUMMY});
3351
3352 # match URL against sane path (can contain "//", see vinfo_child),
3353 # no wildcards
3354 my ($url_prefix, $path) = check_url($vinfo->{URL});
3355 my $url = defined($path) ? lc($url_prefix) . $path : undef;
3356 my $count = 0;
3357 foreach my $ff (@$filter) {
3358 if(defined($ff->{group_eq}) && (grep { $ff->{group_eq} eq $_ } @{$vinfo->{CONFIG}{group}})) {
3359 TRACE "filter \"$ff->{unparsed}\" equals $vinfo->{CONFIG}{CONTEXT} group: $vinfo->{PRINT}" if($do_trace);
3360 return $ff unless($flag_matched);
3361 #push @{$ff->{$flag_matched}}, 'group=' . $ff->{group_eq};
3362 $ff->{$flag_matched} = 1;
3363 $count++;
3364 }
3365 if(defined($ff->{url_regex}) && defined($url) && ($url =~ /$ff->{url_regex}/)) {
3366 TRACE "filter \"$ff->{unparsed}\" matches $vinfo->{CONFIG}{CONTEXT} url: $vinfo->{PRINT}" if($do_trace);
3367 return $ff unless($flag_matched);
3368 #push @{$ff->{$flag_matched}}, $vinfo->{CONFIG}{CONTEXT} . '=' . $vinfo->{PRINT};
3369 $ff->{$flag_matched} = 1;
3370 $count++;
3371 }
3372 if(defined($ff->{host_port_eq}) && defined($vinfo->{HOST})) {
3373 my $host = $ff->{host_port_eq}{host};
3374 my $port = $ff->{host_port_eq}{port};
3375 if((lc($host) eq lc($vinfo->{HOST})) &&
3376 (!defined($port) || (defined($vinfo->{PORT}) && ($port == $vinfo->{PORT}))))
3377 {
3378 TRACE "filter \"$ff->{unparsed}\" matches $vinfo->{CONFIG}{CONTEXT} host: $vinfo->{PRINT}" if($do_trace);
3379 return $ff unless($flag_matched);
3380 #push @{$ff->{$flag_matched}}, $vinfo->{CONFIG}{CONTEXT} . '=' . $vinfo->{PRINT};
3381 $ff->{$flag_matched} = 1;
3382 $count++;
3383 }
3384 }
3385 }
3386 return $count;
3387 }
3388
3389
3390 sub get_related_snapshots($$;$)
3391 {
3392 my $snaproot = shift || die;
3393 my $svol = shift // die;
3394 my $btrbk_basename = shift; # if set, also filter by direct_leaf
3395 my @ret = map( { vinfo_resolved($_, $snaproot, btrbk_direct_leaf => $btrbk_basename) // () }
3396 _related_nodes($svol->{node}, readonly => 1, omit_self => 1) );
3397
3398 if($do_trace) { TRACE "get_related_snapshots: found: $_->{PRINT}" foreach(@ret); }
3399 DEBUG "Found " . scalar(@ret) . " related snapshots of \"$svol->{PRINT}\" in: $snaproot->{PRINT}" . (defined($btrbk_basename) ? "/$btrbk_basename.*" : "");
3400 return @ret;
3401 }
3402
3403
3404 sub _correlated_nodes($$)
3405 {
3406 my $dnode = shift || die; # any node on target filesystem
3407 my $snode = shift || die;
3408 my @ret;
3409
3410 if($snode->{is_root}) {
3411 TRACE "Skip search for correlated targets: source subvolume is btrfs root: " . _fs_path($snode) if($do_trace);
3412 return @ret;
3413 }
3414 unless($snode->{readonly}) {
3415 TRACE "Skip search for correlated targets: source subvolume is not read-only: " . _fs_path($snode) if($do_trace);
3416 return @ret;
3417 }
3418
3419 # find matches by comparing uuid / received_uuid
3420 my $uuid = $snode->{uuid};
3421 my $received_uuid = $snode->{received_uuid};
3422 $received_uuid = undef if($received_uuid eq '-');
3423
3424 my $received_uuid_hash = $dnode->{TREE_ROOT}{RECEIVED_UUID_HASH};
3425 my $uuid_hash = $dnode->{TREE_ROOT}{UUID_HASH};
3426
3427 # match uuid/received_uuid combinations
3428 my @match;
3429 push(@match, @{ $received_uuid_hash->{$uuid} // [] }); # match src.uuid == target.received_uuid
3430 if($received_uuid) {
3431 push(@match, $uuid_hash->{$received_uuid} ); # match src.received_uuid == target.uuid
3432 push(@match, @{ $received_uuid_hash->{$received_uuid} // [] }); # match src.received_uuid == target.received_uuid
3433 }
3434
3435 @ret = grep($_->{readonly}, @match);
3436 TRACE "correlated_nodes: dst=\"" . _fs_path($dnode) . "\", src=\"" . _fs_path($snode) . "\": [" . join(", ", map _fs_path($_),@ret) . "]" if($do_trace);
3437 return @ret;
3438 }
3439
3440
3441 # returns array of vinfo of receive targets matching btrbk name
3442 sub get_receive_targets($$;@)
3443 {
3444 my $droot = shift || die;
3445 my $src_vol = shift || die;
3446 my %opts = @_;
3447 my @ret;
3448
3449 my @correlated = _correlated_nodes($droot->{node}, $src_vol->{node});
3450 my $unexpected;
3451 foreach (@correlated) {
3452 my $vinfo = vinfo_resolved($_, $droot); # returns undef if not below $droot
3453 if(exists($_->{BTRBK_RAW})) {
3454 TRACE "get_receive_targets: found raw receive target: " . _fs_path($_) if($do_trace);
3455 }
3456 elsif($vinfo && ($vinfo->{SUBVOL_PATH} eq $src_vol->{NAME})) { # direct leaf, (SUBVOL_DIR = "", matching NAME)
3457 TRACE "get_receive_targets: found receive target (exact-match): $vinfo->{PRINT}" if($do_trace);
3458 }
3459 elsif($vinfo && (not $opts{exact})) {
3460 TRACE "get_receive_targets: found receive target (non-exact-match): $vinfo->{PRINT}" if($do_trace);
3461 }
3462 else {
3463 TRACE "get_receive_targets: skip unexpected match: " . _fs_path($_) if($do_trace);
3464 $unexpected = 1;
3465 if($opts{warn} && config_key($droot, "warn_unknown_targets")) {
3466 WARN "Receive target of \"$src_vol->{PRINT}\" exists at unknown location: " . ($vinfo ? $vinfo->{PRINT} : _fs_path($_));
3467 }
3468 next;
3469 }
3470 push(@ret, $vinfo);
3471 }
3472 ${$opts{ret_unexpected_only}} = 1 if($opts{ret_unexpected_only} && $unexpected && !scalar(@ret));
3473 return @ret;
3474 }
3475
3476
3477 # returns best correlated receive target within droot (independent of btrbk name)
3478 sub get_best_correlated($$;@)
3479 {
3480 my $droot = shift || die;
3481 my $src_vol = shift || die;
3482 my %opts = @_;
3483 my $inaccessible_nodes = $opts{push_inaccessible_nodes};
3484
3485 my @correlated = _correlated_nodes($droot->{node}, $src_vol->{node}); # all matching src_vol, from droot->TREE_ROOT
3486 foreach (@correlated) {
3487 my $vinfo = vinfo_resolved($_, $droot); # $vinfo is within $droot
3488 return [ $src_vol, $vinfo ] if($vinfo);
3489 }
3490 if($opts{fallback_all_mountpoints}) {
3491 foreach (@correlated) {
3492 my $vinfo = vinfo_resolved_all_mountpoints($_, $droot); # $vinfo is within any mountpoint of filesystem at $droot
3493 return [ $src_vol, $vinfo ] if($vinfo);
3494 }
3495 }
3496 push @$inaccessible_nodes, @correlated if($inaccessible_nodes);
3497 return undef;
3498 }
3499
3500
3501 # returns all related readonly nodes (by parent_uuid relationship), unsorted.
3502 sub _related_nodes($;@)
3503 {
3504 my $snode = shift // die;
3505 my %opts = @_;
3506 TRACE "related_nodes: resolving related subvolumes of: " . _fs_path($snode) if($do_trace);
3507
3508 # iterate parent chain
3509 my @related_nodes;
3510 my $uuid_hash = $snode->{TREE_ROOT}{UUID_HASH};
3511 my $parent_uuid_hash = $snode->{TREE_ROOT}{PARENT_UUID_HASH};
3512 my $node = $snode;
3513 my $uuid = $node->{uuid};
3514 my $abort_distance = 4096;
3515
3516 # climb up parent chain
3517 my $distance = 0; # parent distance
3518 while(($distance < $abort_distance) && defined($node) && ($node->{parent_uuid} ne "-")) {
3519 $uuid = $node->{parent_uuid};
3520 $node = $uuid_hash->{$uuid};
3521 TRACE "related_nodes: d=$distance uuid=$uuid : parent: " . ($node ? _fs_path($node) : "<deleted>") if($do_trace);
3522 $distance++;
3523 }
3524 if($distance >= $abort_distance) {
3525 my $logmsg = "Parent UUID chain exceeds depth=$abort_distance, ignoring related parents of uuid=$uuid for: " . _fs_path($snode);
3526 DEBUG $logmsg;
3527 WARN_ONCE $logmsg unless($opts{nowarn});
3528 }
3529 TRACE "related_nodes: d=$distance uuid=$uuid : top of parent chain" if($do_trace);
3530
3531 # push related children (even if parent node is missing -> siblings)
3532 my @nn;
3533 $abort_distance = $abort_distance;
3534 $distance = $distance * (-1); # child distance (from top parent)
3535 while($uuid) {
3536 push @related_nodes, $node if($node && (!$opts{readonly} || $node->{readonly}));
3537 my $children = $parent_uuid_hash->{$uuid};
3538 if($children) {
3539 if($distance >= $abort_distance) {
3540 my $logmsg = "Parent/child relations exceed depth=$abort_distance, ignoring related children of uuid=$uuid for: " . _fs_path($snode);
3541 DEBUG $logmsg;
3542 WARN_ONCE $logmsg unless($opts{nowarn});
3543 } else {
3544 push @nn, { MARK_UUID => $uuid, MARK_DISTANCE => ($distance + 1) }, @$children;
3545 }
3546 }
3547
3548 if($do_trace) {
3549 if($node) {
3550 if($node->{readonly}) {
3551 TRACE "related_nodes: d=$distance uuid=$uuid : push related readonly: " . _fs_path($node);
3552 } else {
3553 TRACE "related_nodes: d=$distance uuid=$uuid : " . ($opts{readonly} ? "" : "push ") . "related not readonly: " . _fs_path($node);
3554 }
3555 } else {
3556 TRACE "related_nodes: d=$distance uuid=$uuid : related missing: <deleted>";
3557 }
3558 if($children && ($distance < $abort_distance)) {
3559 TRACE "related_nodes: d=$distance uuid=$uuid : postpone " . scalar(@$children) . " children";
3560 }
3561 }
3562
3563 $node = shift @nn;
3564 if(exists($node->{MARK_DISTANCE})) {
3565 # marker reached, restore distance
3566 $distance = $node->{MARK_DISTANCE};
3567 TRACE "related_nodes: d=$distance uuid=$node->{MARK_UUID} : processing children" if($do_trace);
3568 $node = shift @nn;
3569 }
3570 $uuid = $node->{uuid};
3571 }
3572
3573 if($opts{omit_self}) {
3574 my $snode_id = $snode->{id};
3575 my @filtered = grep { $_->{id} != $snode_id } @related_nodes;
3576 TRACE "related_nodes: found total=" . scalar(@filtered) . " related readonly subvolumes" if($do_trace);
3577 return @filtered;
3578 }
3579 TRACE "related_nodes: found total=" . scalar(@related_nodes) . " related readonly subvolumes (including self)" if($do_trace);
3580 return @related_nodes;
3581 }
3582
3583
3584 # returns parent, along with clone sources
3585 sub get_best_parent($$$;@)
3586 {
3587 my $svol = shift // die;
3588 my $snaproot = shift // die;
3589 my $droot = shift || die;
3590 my %opts = @_;
3591 my $ret_clone_src = $opts{clone_src};
3592 my $ret_target_parent_node = $opts{target_parent_node};
3593 my $strict_related = $opts{strict_related};
3594
3595 TRACE "get_best_parent: resolving best common parent for subvolume: $svol->{PRINT} (droot=$droot->{PRINT})" if($do_trace);
3596
3597 # honor incremental_resolve option
3598 my $source_incremental_resolve = config_key($svol, "incremental_resolve");
3599 my $target_incremental_resolve = config_key($droot, "incremental_resolve");
3600 my $resolve_sroot = ($source_incremental_resolve eq "mountpoint") ? $snaproot->{VINFO_MOUNTPOINT} : $snaproot;
3601 my $resolve_droot = ($source_incremental_resolve eq "mountpoint") ? $droot->{VINFO_MOUNTPOINT} : $droot;
3602
3603 # NOTE: Using parents from different mount points does NOT work, see
3604 # <https://github.com/kdave/btrfs-progs/issues/96>.
3605 # btrfs-progs-4.20.2 fails if the parent subvolume is not on same
3606 # mountpoint as the source subvolume:
3607 # - btrfs send -p: "ERROR: not on mount point: /path/to/mountpoint"
3608 # - btrfs receive: "ERROR: parent subvol is not reachable from inside the root subvol"
3609 #
3610 # Note that specifying clones from outside the mount point would work for btrfs send,
3611 # but btrfs receive fails with same error as above (tested with v5.13).
3612 my $source_fallback_all_mountpoints = ($source_incremental_resolve eq "_all_accessible");
3613 my $target_fallback_all_mountpoints = ($target_incremental_resolve eq "_all_accessible");
3614
3615 my @inaccessible_nodes;
3616 my %gbc_opts = ( push_inaccessible_nodes => \@inaccessible_nodes,
3617 fallback_all_mountpoints => $target_fallback_all_mountpoints,
3618 );
3619
3620 # resolve correlated subvolumes by parent_uuid relationship.
3621 # no warnings on aborted search (due to deep relations).
3622 my %c_rel_id; # map id to c_related
3623 my @c_related; # candidates for parent (correlated + related), unsorted
3624 foreach (_related_nodes($svol->{node}, readonly => 1, omit_self => 1, nowarn => 1)) {
3625 my $vinfo = vinfo_resolved($_, $resolve_sroot);
3626 if((not $vinfo) && $source_fallback_all_mountpoints) { # related node is not under $resolve_sroot
3627 $vinfo = vinfo_resolved_all_mountpoints($_, $svol);
3628 }
3629 if($vinfo) {
3630 my $correlated = get_best_correlated($resolve_droot, $vinfo, %gbc_opts);
3631 push @c_related, $correlated if($correlated);
3632 $c_rel_id{$_->{id}} = $correlated;
3633 } else {
3634 DEBUG "Related subvolume is not accessible within $source_incremental_resolve \"$resolve_sroot->{PRINT}\": " . _fs_path($_);
3635 }
3636 }
3637 # sort by cgen
3638 my $cgen_ref = $svol->{node}{readonly} ? $svol->{node}{cgen} : $svol->{node}{gen};
3639
3640 my %c_map; # map correlated candidates to incremental_prefs strategy
3641
3642 # all_related: by parent_uuid relationship, ordered by cgen
3643 $c_map{aro} = [ sort { ($cgen_ref - $a->[0]{node}{cgen}) <=> ($cgen_ref - $b->[0]{node}{cgen}) }
3644 grep { $_->[0]{node}{cgen} <= $cgen_ref } @c_related ];
3645 $c_map{arn} = [ sort { ($a->[0]{node}{cgen} - $cgen_ref) <=> ($b->[0]{node}{cgen} - $cgen_ref) }
3646 grep { $_->[0]{node}{cgen} > $cgen_ref } @c_related ];
3647
3648 # NOTE: While _related_nodes() returns deep parent_uuid
3649 # relations, there is always a chance that these relations get
3650 # broken.
3651 #
3652 # Consider parent_uuid chain ($svol readonly)
3653 # B->A, C->B, delete B: C has no relation to A.
3654 # This is especially true for backups and archives (btrfs receive)
3655 #
3656 # For snapshots (here: S=$svol readwrite) the scenario is different:
3657 # A->S, B->S, C->S, delete B: A still has a relation to C.
3658 #
3659 # resolve correlated subvolumes in same directory matching btrbk file name scheme
3660 if(exists($svol->{node}{BTRBK_BASENAME})) {
3661 my $snaproot_btrbk_direct_leaf = vinfo_subvol_list($snaproot, readonly => 1, btrbk_direct_leaf => $svol->{node}{BTRBK_BASENAME});
3662 my @sbdl_older = sort { cmp_date($b->{node}{BTRBK_DATE}, $a->{node}{BTRBK_DATE}) }
3663 grep { cmp_date($_->{node}{BTRBK_DATE}, $svol->{node}{BTRBK_DATE}) < 0 } @$snaproot_btrbk_direct_leaf;
3664 my @sbdl_newer = sort { cmp_date($a->{node}{BTRBK_DATE}, $b->{node}{BTRBK_DATE}) }
3665 grep { cmp_date($_->{node}{BTRBK_DATE}, $svol->{node}{BTRBK_DATE}) > 0 } @$snaproot_btrbk_direct_leaf;
3666
3667 # snapdir_all: btrbk_direct_leaf, ordered by btrbk timestamp
3668 $c_map{sao} = [ map { $c_rel_id{$_->{node}{id}} // get_best_correlated($resolve_droot, $_, %gbc_opts) // () } @sbdl_older ];
3669 $c_map{san} = [ map { $c_rel_id{$_->{node}{id}} // get_best_correlated($resolve_droot, $_, %gbc_opts) // () } @sbdl_newer ];
3670
3671 # snapdir_related: btrbk_direct_leaf with parent_uuid relationship, ordered by btrbk timestamp
3672 $c_map{sro} = [ map { $c_rel_id{$_->{node}{id}} // () } @sbdl_older ];
3673 $c_map{srn} = [ map { $c_rel_id{$_->{node}{id}} // () } @sbdl_newer ];
3674 }
3675
3676 if(scalar @inaccessible_nodes) { # populated by get_best_correlated()
3677 WARN "Best common parent for \"$svol->{PRINT}\" is not accessible within target $target_incremental_resolve \"$resolve_droot->{PRINT}\", ignoring: " . join(", ", map('"' . _fs_path($_) . '"',@inaccessible_nodes));
3678 }
3679
3680 # resolve parent (and required clone sources) according to incremental_prefs
3681
3682 if($do_trace) {
3683 TRACE "get_best_parent: related reference cgen=$svol->{node}{cgen}";
3684 foreach my $search (@incremental_prefs_avail) {
3685 TRACE map("get_best_parent: ${search}: $_->[0]{PRINT} (cgen=$_->[0]{node}{cgen}) $_->[1]{PRINT}", @{$c_map{$search}});
3686 }
3687 }
3688
3689 my @parent;
3690 my @isk = map { $_ eq "defaults" ? @incremental_prefs_default : $_ } @{config_key($svol, "incremental_prefs")};
3691 foreach(@isk) {
3692 TRACE "processing incremental_prefs: $_";
3693 my ($k, $n) = split /:/;
3694 my $c_list = $c_map{$k} // next;
3695 for(1 .. ($n // @$c_list)) {
3696 my $cc = shift @$c_list // last;
3697 next if(grep { $_->[0]{node}{id} == $cc->[0]{node}{id} } @parent);
3698 DEBUG "Resolved " . (@parent ? "clone source" : "parent") . " (" .
3699 "next closest " . ($k =~ /n/ ? "newer" : "older") .
3700 " by " . ($k =~ /s/ ? "btrbk timestamp in snapdir" : "cgen") .
3701 ", " . ($k =~ /r/ ? "with" : "regardless of") . " parent_uuid relationship" .
3702 "): $cc->[0]{PRINT}" if($loglevel >= 3);
3703 push @parent, $cc;
3704 }
3705 }
3706
3707 # assemble results
3708 unless(scalar @parent) {
3709 DEBUG("No suitable common parents of \"$svol->{PRINT}\" found in src=\"$resolve_sroot->{PRINT}/\", target=\"$resolve_droot->{PRINT}/\"");
3710 return undef;
3711 }
3712
3713 if($strict_related && (!grep(exists($c_rel_id{$_->[0]{node}{id}}), @parent))) {
3714 # no relations by parent_uuid found
3715 WARN "No related common parent found (by parent_uuid relationship) for: $svol->{PRINT}",
3716 "Hint: setting option \"incremental\" to \"yes\" (instead of \"strict\") will use parent: " . join(", ", map $_->[0]{PRINT}, @parent);
3717 return undef;
3718 }
3719
3720 my $ret_parent = shift @parent;
3721 $$ret_clone_src = [ map $_->[0], @parent ] if($ret_clone_src);
3722 $$ret_target_parent_node = $ret_parent->[1]{node} if($ret_target_parent_node);
3723 return $ret_parent->[0];
3724 }
3725
3726
3727 sub get_latest_related_snapshot($$;$)
3728 {
3729 my $sroot = shift || die;
3730 my $svol = shift // die;
3731 my $btrbk_basename = shift;
3732 my $latest = undef;
3733 my $gen = -1;
3734 foreach (get_related_snapshots($sroot, $svol, $btrbk_basename)) {
3735 if($_->{node}{cgen} > $gen) {
3736 $latest = $_;
3737 $gen = $_->{node}{cgen};
3738 }
3739 }
3740 if($latest) {
3741 DEBUG "Latest snapshot child for \"$svol->{PRINT}#$svol->{node}{gen}\" is: $latest->{PRINT}#$latest->{node}{cgen}";
3742 } else {
3743 DEBUG "No latest snapshots found for: $svol->{PRINT}";
3744 }
3745 return $latest;
3746 }
3747
3748
3749 sub check_file($$;@)
3750 {
3751 my $file = shift // die;
3752 my $accept = shift || die;
3753 my %opts = @_;
3754 my $sanitize = $opts{sanitize};
3755 my $error_statement = $opts{error_statement}; # if not defined, no error messages are printed
3756
3757 if($accept->{absolute} && $accept->{relative}) {
3758 # accepted, matches either absolute or relative
3759 }
3760 elsif($accept->{absolute}) {
3761 unless($file =~ /^\//) {
3762 ERROR "Only absolute files allowed $error_statement" if(defined($error_statement));
3763 return undef;
3764 }
3765 }
3766 elsif($accept->{relative}) {
3767 if($file =~ /^\//) {
3768 ERROR "Only relative files allowed $error_statement" if(defined($error_statement));
3769 return undef;
3770 }
3771 }
3772 elsif($accept->{name_only}) {
3773 if($file =~ /\//) {
3774 ERROR "Invalid file name ${error_statement}: $file" if(defined($error_statement));
3775 return undef;
3776 }
3777 }
3778 elsif(not $accept->{wildcards}) {
3779 die("accept_type must contain either 'relative' or 'absolute'");
3780 }
3781
3782 if($file =~ /\n/) {
3783 ERROR "Unsupported newline in file ${error_statement}: " . ($file =~ s/\n/\\n/gr) if(defined($error_statement));
3784 return undef;
3785 }
3786 if(($file =~ /^\.\.$/) || ($file =~ /^\.\.\//) || ($file =~ /\/\.\.\//) || ($file =~ /\/\.\.$/)) {
3787 ERROR "Illegal directory traversal ${error_statement}: $file" if(defined($error_statement));
3788 return undef;
3789 }
3790 if($sanitize) {
3791 $file =~ s/^\s+//;
3792 $file =~ s/\s+$//;
3793 $file =~ s/\/(\.?\/)+/\//g; # sanitize "//", "/./" -> "/"
3794 $file =~ s/\/\.$/\//; # sanitize trailing "/." -> "/"
3795 $file =~ s/\/$// unless($file eq '/'); # remove trailing slash
3796 }
3797 elsif(($file =~ /^\s/) || ($file =~ /\s$/)) {
3798 ERROR "Illegal leading/trailing whitespace ${error_statement}: \"$file\"" if(defined($error_statement));
3799 return undef;
3800 }
3801 if($safe_commands && $file !~ /^$safe_file_match$/) {
3802 ERROR "Invalid file name (restricted by \"safe_commands\" option) ${error_statement}: \"$file\"" if(defined($error_statement));
3803 return undef;
3804 }
3805
3806 return $file;
3807 }
3808
3809
3810 sub check_url($;@)
3811 {
3812 my $url = shift // die;
3813 my %opts = @_;
3814 my $url_prefix = "";
3815
3816 if($url =~ /^ssh:\/\//) {
3817 if($url =~ s/^(ssh:\/\/($host_name_match|$ipv4_addr_match|\[$ipv6_addr_match\])(:[1-9][0-9]*)?)\//\//) {
3818 $url_prefix = $1;
3819 }
3820 }
3821 elsif($url =~ s/^($host_name_match|$ipv4_addr_match|\[$ipv6_addr_match\]):\//\//) {
3822 # convert "my.host.com:/my/path", "[2001:db8::7]:/my/path" to ssh url
3823 $url_prefix = "ssh://" . $1;
3824 }
3825 # if no url prefix match, treat it as file and let check_file() print errors
3826
3827 return ( $url_prefix, check_file($url, { absolute => 1, wildcards => $opts{accept_wildcards} }, sanitize => 1, %opts) );
3828 }
3829
3830
3831 sub config_key($$;$)
3832 {
3833 my $config = shift || die;
3834 my $key = shift || die;
3835 my $match = shift;
3836 $config = $config->{CONFIG} if($config->{CONFIG}); # accept vinfo for $config
3837
3838 my $val;
3839 if(exists($config_override{$key})) {
3840 TRACE "config_key: OVERRIDE key=$key to value=" . ($config_override{$key} // "<undef>") if($do_trace);
3841 $val = $config_override{$key};
3842 }
3843 else {
3844 while(not exists($config->{$key})) {
3845 # note: while all config keys exist in "meta" context (at least with default values),
3846 # we also allow fake configs (CONTEXT="cmdline") which have no PARENT.
3847 return undef unless($config->{PARENT});
3848 $config = $config->{PARENT};
3849 }
3850 $val = $config->{$key};
3851 }
3852 return undef unless defined($val);
3853 return $val unless defined($match);
3854
3855 if(ref($val) eq "ARRAY") {
3856 return grep(/^$match$/, @$val) ? $match : undef;
3857 } else {
3858 return ($val eq $match) ? $match : undef;
3859 }
3860 }
3861
3862
3863 sub config_key_lru($$;$) {
3864 my $vinfo = shift || die;
3865 my $key = shift || die;
3866 my $match = shift;
3867 my $retval;
3868 if(defined($vinfo->{HOST})) {
3869 $retval //= config_key($vinfo, $key . "_remote", $match);
3870 } else {
3871 $retval //= config_key($vinfo, $key . "_local_user", $match) if($>); # $EUID, $EFFECTIVE_USER_ID
3872 $retval //= config_key($vinfo, $key . "_local", $match);
3873 }
3874 $retval //= config_key($vinfo, $key, $match);
3875 return $retval;
3876 }
3877
3878
3879 sub config_preserve_hash($$;@)
3880 {
3881 my $config = shift || die;
3882 my $prefix = shift || die;
3883 my %opts = @_;
3884 if($opts{wipe}) {
3885 return { hod => 0, dow => 'sunday', min => 'latest', min_q => 'latest' };
3886 }
3887 my $preserve = config_key($config, $prefix . "_preserve") // {};
3888 my %ret = ( %$preserve, # make a copy (don't pollute config)
3889 hod => config_key($config, "preserve_hour_of_day"),
3890 dow => config_key($config, "preserve_day_of_week")
3891 );
3892 my $preserve_min = config_key($config, $prefix . "_preserve_min");
3893 if(defined($preserve_min)) {
3894 $ret{min} = $preserve_min; # used for raw schedule output
3895 if(($preserve_min eq 'all') || ($preserve_min eq 'latest')) {
3896 $ret{min_q} = $preserve_min;
3897 }
3898 elsif($preserve_min =~ /^([0-9]+)([hdwmy])$/) {
3899 $ret{min_n} = $1;
3900 $ret{min_q} = $2;
3901 }
3902 else { die; }
3903 }
3904 return \%ret;
3905 }
3906
3907
3908 sub config_compress_hash($$)
3909 {
3910 my $config = shift || die;
3911 my $config_key = shift || die;
3912 my $compress_key = config_key($config, $config_key);
3913 return undef unless($compress_key);
3914 return {
3915 key => $compress_key,
3916 level => config_key($config, $config_key . "_level"),
3917 long => config_key($config, $config_key . "_long"),
3918 threads => config_key($config, $config_key . "_threads"),
3919 adapt => config_key($config, $config_key . "_adapt"),
3920 };
3921 }
3922
3923
3924 sub config_stream_hash($$)
3925 {
3926 my $source = shift || die;
3927 my $target = shift || die;
3928 return {
3929 stream_compress => config_compress_hash($target, "stream_compress"),
3930
3931 # for remote source, limits read rate of ssh stream output after decompress
3932 # for remote target, limits read rate of "btrfs send"
3933 # for both local, limits read rate of "btrfs send"
3934 # for raw targets, limits read rate of "btrfs send | xz" (raw_target_compress)
3935 local_sink => {
3936 stream_buffer => config_key($target, "stream_buffer"),
3937 rate_limit => config_key($target, "rate_limit"),
3938 show_progress => $show_progress,
3939 },
3940
3941 # limits read rate of "btrfs send"
3942 rsh_source => { # limit read rate after "btrfs send", before compression
3943 stream_buffer => config_key($source, "stream_buffer_remote"),
3944 rate_limit => config_key($source, "rate_limit_remote"),
3945 #rate_limit_out => config_key($source, "rate_limit_remote"), # limit write rate
3946 },
3947
3948 # limits read rate of ssh stream output
3949 rsh_sink => {
3950 stream_buffer => config_key($target, "stream_buffer_remote"),
3951 rate_limit => config_key($target, "rate_limit_remote"),
3952 #rate_limit_in => config_key($target, "rate_limit_remote"),
3953 },
3954 };
3955 }
3956
3957
3958 sub config_encrypt_hash($$)
3959 {
3960 my $config = shift || die;
3961 my $config_key = shift || die;
3962 my $encrypt_type = config_key($config, $config_key);
3963 return undef unless($encrypt_type);
3964 return {
3965 type => $encrypt_type,
3966 keyring => config_key($config, "gpg_keyring"),
3967 recipient => config_key($config, "gpg_recipient"),
3968 iv_size => config_key($config, "openssl_iv_size"),
3969 ciphername => config_key($config, "openssl_ciphername"),
3970 keyfile => config_key($config, "openssl_keyfile"),
3971 kdf_keygen_each => (config_key($config, "kdf_keygen") eq "each"),
3972 kdf_backend => config_key($config, "kdf_backend"),
3973 kdf_keysize => config_key($config, "kdf_keysize"),
3974 };
3975 }
3976
3977
3978 sub config_dump_keys($;@)
3979 {
3980 my $config = shift || die;
3981 my %opts = @_;
3982 my @ret;
3983 my $maxlen = 0;
3984 $config = $config->{CONFIG} if($config->{CONFIG}); # accept vinfo for $config
3985
3986 foreach my $key (sort keys %config_options)
3987 {
3988 my $val;
3989 next if($config_options{$key}->{deprecated}{DEFAULT});
3990 next unless($opts{all} || exists($config->{$key}) || exists($config_override{$key}));
3991 next if($config_options{$key}{context} && !grep(/^$config->{CONTEXT}$/, @{$config_options{$key}{context}}));
3992 $val = config_key($config, $key);
3993
3994 my @va = (ref($val) eq "ARRAY") ? ($config_options{$key}->{split} ? join(" ", @$val) : @$val) : $val;
3995 foreach(@va) {
3996 if(defined($_)) {
3997 if($config_options{$key}->{accept_preserve_matrix}) {
3998 $_ = format_preserve_matrix($_, format => "config");
3999 }
4000 }
4001 $_ //= grep(/^no$/, @{$config_options{$key}{accept} // []}) ? "no" : "<unset>";
4002 my $comment = $_ eq "<unset>" ? "# " : "";
4003 my $len = length($key);
4004 $maxlen = $len if($len > $maxlen);
4005 push @ret, { comment => $comment, key => $key, val => $_, len => $len };
4006 }
4007 }
4008 return map { ($opts{prefix} // "") . $_->{comment} . $_->{key} . (' ' x (1 + $maxlen - $_->{len})) . ' ' . $_->{val} } @ret;
4009 }
4010
4011
4012 sub append_config_option($$$$;@)
4013 {
4014 my $config = shift;
4015 my $key = shift;
4016 my $value = shift;
4017 my $context = shift;
4018 my %opts = @_;
4019 my $error_statement = $opts{error_statement} // "";
4020
4021 my $opt = $config_options{$key};
4022
4023 # accept only keys listed in %config_options
4024 unless($opt) {
4025 ERROR "Unknown option \"$key\" $error_statement";
4026 return undef;
4027 }
4028
4029 if($opt->{context} && !grep(/^$context$/, @{$opt->{context}}) && ($context ne "OVERRIDE")) {
4030 ERROR "Option \"$key\" is only allowed in " . join(" or ", @{$opt->{context}}) . " context $error_statement";
4031 return undef;
4032 }
4033
4034 if($opt->{deny_glob_context} && $config->{GLOB_CONTEXT}) {
4035 ERROR "Option \"$key\" is not allowed on section with wildcards $error_statement";
4036 return undef;
4037 }
4038
4039 my $ovalue = $value;
4040
4041 if($value eq "") {
4042 $value = "yes";
4043 TRACE "option \"$key\" has no value, setting to \"yes\"" if($do_trace);
4044 }
4045
4046 if($opt->{split}) {
4047 $value = [ split($config_split_match, $value) ];
4048 }
4049
4050 my $accepted;
4051 if($opt->{accept}) {
4052 $accepted = 1;
4053 foreach my $val (ref($value) ? @$value : $value) {
4054 $accepted = 0, last unless(grep { $val =~ /^$_$/ } @{$opt->{accept}});
4055 TRACE "option \"$key=$val\" found in accept list" if($do_trace);
4056 }
4057 }
4058 if(!$accepted && $opt->{accept_file}) {
4059 # be very strict about file options, for security sake
4060 $value = check_file($value, $opt->{accept_file}, sanitize => 1, error_statement => ($error_statement ? "for option \"$key\" $error_statement" : undef));
4061 return undef unless(defined($value));
4062
4063 TRACE "option \"$key=$value\" is a valid file, accepted" if($do_trace);
4064 $value = "no" if($value eq "."); # maps to undef later
4065 $accepted = 1;
4066 }
4067 if(!$accepted && $opt->{accept_preserve_matrix}) {
4068 my %preserve;
4069 my $s = ' ' . $value;
4070 while($s =~ s/\s+(\*|[0-9]+)([hdwmyHDWMY])//) {
4071 my $n = $1;
4072 my $q = lc($2); # qw( h d w m y )
4073 $n = 'all' if($n eq '*');
4074 if(exists($preserve{$q})) {
4075 ERROR "Value \"$value\" failed input validation for option \"$key\": multiple definitions of '$q' $error_statement";
4076 return undef;
4077 }
4078 $preserve{$q} = $n;
4079 }
4080 unless($s eq "") {
4081 ERROR "Value \"$value\" failed input validation for option \"$key\" $error_statement";
4082 return undef;
4083 }
4084 TRACE "adding preserve matrix $context context:" . Data::Dumper->new([\%preserve], [ $key ])->Indent(0)->Pad(' ')->Quotekeys(0)->Pair('=>')->Dump() if($do_trace && $do_dumper);
4085 $config->{$key} = \%preserve;
4086 return $config;
4087 }
4088 if(!$accepted) {
4089 if($ovalue eq "") {
4090 ERROR "Unsupported empty value for option \"$key\" $error_statement";
4091 } else {
4092 ERROR "Unsupported value \"$ovalue\" for option \"$key\" $error_statement";
4093 }
4094 return undef;
4095 }
4096
4097 if($opt->{require_bin} && (not check_exe($opt->{require_bin}))) {
4098 WARN "Found option \"$key\", but required executable \"$opt->{require_bin}\" does not exist on your system. Please install \"$opt->{require_bin}\".";
4099 WARN "Ignoring option \"$key\" $error_statement";
4100 $value = "no";
4101 }
4102
4103 if($opt->{deprecated}) {
4104 my $dh = $opt->{deprecated}{$value} // $opt->{deprecated}{DEFAULT} // {};
4105 $dh = $opt->{deprecated}{MATCH} if($opt->{deprecated}{MATCH} && ($value =~ $opt->{deprecated}{MATCH}{regex}));
4106 if($dh->{ABORT}) {
4107 ERROR "Deprecated (incompatible) option \"$key\" found $error_statement, refusing to continue", $dh->{warn};
4108 return undef;
4109 }
4110 my @wmsg;
4111 push @wmsg, "Found deprecated option \"$key $value\" $error_statement", $dh->{warn} if($dh->{warn});
4112 if(defined($dh->{replace_key})) {
4113 $key = $dh->{replace_key};
4114 $value = $dh->{replace_value};
4115 push @wmsg, "Using \"$key $value\"";
4116 }
4117 WARN @wmsg if(@wmsg);
4118 if($dh->{FAILSAFE_PRESERVE}) {
4119 unless($config_override{FAILSAFE_PRESERVE}) { # warn only once
4120 WARN "Entering failsafe mode:";
4121 WARN " - preserving ALL snapshots for ALL subvolumes";
4122 WARN " - ignoring ALL targets (skipping backup creation)";
4123 WARN " - please read \"doc/upgrade_to_v0.23.0.md\"";
4124 $config_override{FAILSAFE_PRESERVE} = "Failsafe mode active (deprecated configuration)";
4125 }
4126 $config_override{snapshot_preserve_min} = 'all';
4127 return $config;
4128 }
4129 }
4130
4131 if($opt->{allow_multiple}) {
4132 my $aref = $config->{$key} // [];
4133 my @val = ref($value) ? @$value : $value;
4134 push(@$aref, @val);
4135 TRACE "pushing option \"$key=[" . join(",", @val) . "]\" to $aref=[" . join(',', @$aref) . "]" if($do_trace);
4136 $value = $aref;
4137 }
4138 elsif(exists($config->{$key})) {
4139 unless($opt->{c_default}) { # note: computed defaults are already present
4140 WARN "Option \"$key\" redefined $error_statement";
4141 }
4142 }
4143
4144 TRACE "adding option \"$key=$value\" to $context context" if($do_trace);
4145 $value = undef if($value eq "no"); # we don't want to check for "no" all the time
4146 $config->{$key} = $value;
4147 return $config;
4148 }
4149
4150
4151 sub parse_config_line($$$;@)
4152 {
4153 my ($cur, $key, $value, %opts) = @_;
4154 my $root = $cur;
4155 $root = $root->{PARENT} while($root->{CONTEXT} ne "global");
4156 my $error_statement = $opts{error_statement} // "";
4157
4158 if($key eq "volume")
4159 {
4160 $value =~ s/^"(.*)"$/$1/;
4161 $value =~ s/^'(.*)'$/$1/;
4162
4163 $cur = $root;
4164 TRACE "config: context forced to: $cur->{CONTEXT}" if($do_trace);
4165
4166 # be very strict about file options, for security sake
4167 my ($url_prefix, $path) = check_url($value, error_statement => "for option \"$key\" $error_statement");
4168 return undef unless(defined($path));
4169 TRACE "config: adding volume \"$url_prefix$path\" to global context" if($do_trace);
4170 die unless($cur->{CONTEXT} eq "global");
4171 my $volume = { CONTEXT => "volume",
4172 PARENT => $cur,
4173 SUBSECTION => [],
4174 url => $url_prefix . $path,
4175 };
4176 push(@{$cur->{SUBSECTION}}, $volume);
4177 $cur = $volume;
4178 }
4179 elsif($key eq "subvolume")
4180 {
4181 $value =~ s/^"(.*)"$/$1/;
4182 $value =~ s/^'(.*)'$/$1/;
4183
4184 while($cur->{CONTEXT} ne "volume") {
4185 if($cur->{CONTEXT} eq "global") {
4186 TRACE "config: adding dummy volume context" if($do_trace);
4187 my $volume = { CONTEXT => "volume",
4188 PARENT => $cur,
4189 SUBSECTION => [],
4190 DUMMY => 1,
4191 url => "/dev/null",
4192 };
4193 push(@{$cur->{SUBSECTION}}, $volume);
4194 $cur = $volume;
4195 last;
4196 }
4197 $cur = $cur->{PARENT} || die;
4198 TRACE "config: context changed to: $cur->{CONTEXT}" if($do_trace);
4199 }
4200 # be very strict about file options, for security sake
4201 my $url;
4202 if(!$cur->{DUMMY} && (my $rel_path = check_file($value, { relative => 1, wildcards => 1 }, sanitize => 1))) {
4203 $url = ($rel_path eq '.') ? $cur->{url} : $cur->{url} . '/' . $rel_path;
4204 }
4205 else {
4206 my ($url_prefix, $path) = check_url($value, accept_wildcards => 1, error_statement => "for option \"$key\"" . ($cur->{DUMMY} ? " (if no \"volume\" section is declared)" : "") . " $error_statement");
4207 return undef unless(defined($path));
4208 $url = $url_prefix . $path;
4209 }
4210
4211 # snapshot_name defaults to subvolume name (or volume name if subvolume=".")
4212 my $default_snapshot_name = $url;
4213 $default_snapshot_name =~ s/^.*\///;
4214 $default_snapshot_name = 'ROOT' if($default_snapshot_name eq ""); # if volume="/"
4215
4216 TRACE "config: adding subvolume \"$url\" to volume context: $cur->{url}" if($do_trace);
4217 my $subvolume = { CONTEXT => "subvolume",
4218 PARENT => $cur,
4219 # SUBSECTION => [], # handled by target propagation
4220 url => $url,
4221 snapshot_name => $default_snapshot_name, # computed default (c_default)
4222 };
4223 $subvolume->{GLOB_CONTEXT} = 1 if($value =~ /\*/);
4224 push(@{$cur->{SUBSECTION}}, $subvolume);
4225 $cur = $subvolume;
4226 }
4227 elsif($key eq "target")
4228 {
4229 if($cur->{CONTEXT} eq "target") {
4230 $cur = $cur->{PARENT} || die;
4231 TRACE "config: context changed to: $cur->{CONTEXT}" if($do_trace);
4232 }
4233
4234 # As of btrbk-0.28.0, target_type is optional and defaults to "send-receive"
4235 my $target_type = $config_target_types[0];
4236 $target_type = lc($1) if($value =~ s/^([a-zA-Z_-]+)\s+//);
4237 unless(grep(/^\Q$target_type\E$/, @config_target_types)) {
4238 ERROR "Unknown target type \"$target_type\" $error_statement";
4239 return undef;
4240 }
4241
4242 $value =~ s/^"(.*)"$/$1/;
4243 $value =~ s/^'(.*)'$/$1/;
4244
4245 my ($url_prefix, $path) = check_url($value, error_statement => "for option \"$key\" $error_statement");
4246 return undef unless(defined($path));
4247
4248 TRACE "config: adding target \"$url_prefix$path\" (type=$target_type) to $cur->{CONTEXT} context" . ($cur->{url} ? ": $cur->{url}" : "") if($do_trace);
4249 my $target = { CONTEXT => "target",
4250 PARENT => $cur,
4251 target_type => $target_type,
4252 url => $url_prefix . $path,
4253 };
4254 # NOTE: target sections are propagated to the apropriate SUBSECTION in _config_propagate_target()
4255 $cur->{TARGET} //= [];
4256 push(@{$cur->{TARGET}}, $target);
4257 $cur = $target;
4258 }
4259 else
4260 {
4261 $value =~ s/^"(.*)"$/$1/;
4262 $value =~ s/^'(.*)'$/$1/;
4263 return append_config_option($cur, $key, $value, $cur->{CONTEXT}, error_statement => $error_statement);
4264 }
4265
4266 return $cur;
4267 }
4268
4269
4270 sub _config_propagate_target
4271 {
4272 my $cur = shift;
4273 foreach my $subsection (@{$cur->{SUBSECTION}}) {
4274 my @propagate_target;
4275 foreach my $target (@{$cur->{TARGET}}) {
4276 TRACE "propagating target \"$target->{url}\" from $cur->{CONTEXT} context to: $subsection->{CONTEXT} $subsection->{url}" if($do_trace);
4277 die if($target->{SUBSECTION});
4278
4279 # don't propagate if a target of same target_type and url already exists in subsection
4280 if($subsection->{TARGET} &&
4281 grep({ ($_->{url} eq $target->{url}) && ($_->{target_type} eq $target->{target_type}) } @{$subsection->{TARGET}}))
4282 {
4283 DEBUG "Skip propagation of \"target $target->{target_type} $target->{url}\" from $cur->{CONTEXT} context to \"$subsection->{CONTEXT} $subsection->{url}\": same target already exists";
4284 next;
4285 }
4286
4287 my %copy = ( %$target, PARENT => $subsection );
4288 push @propagate_target, \%copy;
4289 }
4290 $subsection->{TARGET} //= [];
4291 unshift @{$subsection->{TARGET}}, @propagate_target; # maintain config order: propagated targets go in front of already defined targets
4292 if($subsection->{CONTEXT} eq "subvolume") {
4293 # finally create missing SUBSECTION in subvolume context
4294 die if($subsection->{SUBSECTION});
4295 $subsection->{SUBSECTION} = $subsection->{TARGET};
4296 }
4297 else {
4298 # recurse into SUBSECTION
4299 _config_propagate_target($subsection);
4300 }
4301 }
4302 delete $cur->{TARGET};
4303 return $cur;
4304 }
4305
4306
4307 sub _config_collect_values
4308 {
4309 my $config = shift;
4310 my $key = shift;
4311 my @values;
4312 push(@values, @{$config->{$key}}) if(ref($config->{$key}) eq "ARRAY");
4313 foreach (@{$config->{SUBSECTION}}) {
4314 push(@values, _config_collect_values($_, $key));
4315 }
4316 return @values;
4317 }
4318
4319
4320 sub init_config(@)
4321 {
4322 my %defaults = ( CONTEXT => "meta", @_ );
4323 # set defaults
4324 foreach (keys %config_options) {
4325 next if $config_options{$_}->{deprecated}; # don't pollute hash with deprecated options
4326 $defaults{$_} = $config_options{$_}->{default};
4327 }
4328 return { CONTEXT => "global", SUBSECTION => [], PARENT => \%defaults };
4329 }
4330
4331
4332 sub _config_file(@) {
4333 my @config_files = @_;
4334 foreach my $file (@config_files) {
4335 TRACE "config: checking for file: $file" if($do_trace);
4336 return $file if(-r "$file");
4337 }
4338 return undef;
4339 }
4340
4341 sub parse_config($)
4342 {
4343 my $file = shift;
4344 return undef unless($file);
4345
4346 my $root = init_config(SRC_FILE => $file);
4347 my $cur = $root;
4348
4349 TRACE "config: open configuration file: $file" if($do_trace);
4350 open(FILE, '<', $file) or die $!;
4351 while (<FILE>) {
4352 chomp;
4353 s/((?:[^"'#]*(?:"[^"]*"|'[^']*'))*[^"'#]*)#.*/$1/; # remove comments
4354 next if /^\s*$/; # ignore empty lines
4355 s/^\s*//; # remove leading whitespace
4356 s/\s*$//; # remove trailing whitespace
4357 TRACE "config: parsing line $. with context=$cur->{CONTEXT}: \"$_\"" if($do_trace);
4358 unless(/^([a-zA-Z_]+)(?:\s+(.*))?$/) {
4359 ERROR "Parse error in \"$file\" line $.";
4360 $root = undef;
4361 last;
4362 }
4363 unless($cur = parse_config_line($cur, lc($1), $2 // "", error_statement => "in \"$file\" line $.")) {
4364 $root = undef;
4365 last;
4366 }
4367 TRACE "line processed: new context=$cur->{CONTEXT}" if($do_trace);
4368 }
4369 close FILE || ERROR "Failed to close configuration file: $!";
4370
4371 _config_propagate_target($root);
4372
4373 return $root;
4374 }
4375
4376
4377 # sets $target->{CONFIG}->{ABORTED} on failure
4378 # sets $target->{SUBVOL_RECEIVED}
4379 sub macro_send_receive(@)
4380 {
4381 my %info = @_;
4382 my $source = $info{source} || die;
4383 my $target = $info{target} || die;
4384 my $parent = $info{parent};
4385 my @clone_src = @{ $info{clone_src} // [] }; # copy array
4386 my $config_target = $target->{CONFIG};
4387 my $target_type = $config_target->{target_type} || die;
4388 my $incremental = config_key($config_target, "incremental");
4389
4390 # check for existing target subvolume
4391 if(my $err_vol = vinfo_subvol($target, $source->{NAME})) {
4392 my $err_msg = "Please delete stray subvolume: \"btrfs subvolume delete $err_vol->{PRINT}\"";
4393 ABORTED($config_target, "Target subvolume \"$err_vol->{PRINT}\" already exists");
4394 FIX_MANUALLY($config_target, $err_msg);
4395 ERROR ABORTED_TEXT($config_target) . ", aborting send/receive of: $source->{PRINT}";
4396 ERROR $err_msg;
4397 return undef;
4398 }
4399
4400 if($incremental)
4401 {
4402 # create backup from latest common
4403 if($parent) {
4404 INFO "Creating incremental backup...";
4405 }
4406 elsif($incremental ne "strict") {
4407 INFO "No common parent subvolume present, creating non-incremental backup...";
4408 }
4409 else {
4410 WARN "Backup to $target->{PRINT} failed: no common parent subvolume found for \"$source->{PRINT}\", and option \"incremental\" is set to \"strict\"";
4411 ABORTED($config_target, "No common parent subvolume found, and option \"incremental\" is set to \"strict\"");
4412 return undef;
4413 }
4414 unless(config_key($target, "incremental_clones")) {
4415 INFO "Ignoring " . scalar(@clone_src) . " clone sources (incremental_clones=no)" if(@clone_src);
4416 @clone_src = ();
4417 delete $info{clone_src};
4418 }
4419 }
4420 else {
4421 INFO "Creating non-incremental backup...";
4422 $parent = undef;
4423 @clone_src = ();
4424 delete $info{parent};
4425 delete $info{clone_src};
4426 }
4427
4428 my $ret;
4429 my $vol_received;
4430 my $raw_info;
4431 if($target_type eq "send-receive")
4432 {
4433 $ret = btrfs_send_receive($source, $target, $parent, \@clone_src, \$vol_received);
4434 ABORTED($config_target, "Failed to send/receive subvolume") unless($ret);
4435 }
4436 elsif($target_type eq "raw")
4437 {
4438 unless($dryrun) {
4439 # make sure we know the source uuid
4440 if($source->{node}{uuid} =~ /^$fake_uuid_prefix/) {
4441 DEBUG "Fetching uuid of new subvolume: $source->{PRINT}";
4442 my $detail = btrfs_subvolume_show($source);
4443 return undef unless($detail);
4444 die unless($detail->{uuid});
4445 $source->{node}{uuid} = $detail->{uuid};
4446 $uuid_cache{$detail->{uuid}} = $source->{node};
4447 }
4448 }
4449 $ret = btrfs_send_to_file($source, $target, $parent, \$vol_received, \$raw_info);
4450 ABORTED($config_target, "Failed to send subvolume to raw file") unless($ret);
4451 }
4452 else
4453 {
4454 die "Illegal target type \"$target_type\"";
4455 }
4456
4457 # inject fake vinfo
4458
4459 # NOTE: it's not possible to add (and compare) correct target $detail
4460 # from btrfs_send_receive(), as source detail also has fake uuid.
4461 if($ret) {
4462 vinfo_inject_child($target, $vol_received, {
4463 # NOTE: this is not necessarily the correct parent_uuid (on
4464 # receive, btrfs-progs picks the uuid of the first (lowest id)
4465 # matching possible parent), whereas the target_parent is the
4466 # first from _correlated_nodes().
4467 #
4468 # NOTE: the parent_uuid of an injected receive target is not used
4469 # anywhere in btrbk at the time of writing
4470 parent_uuid => $parent ? $info{target_parent_node}->{uuid} : '-',
4471 received_uuid => $source->{node}{received_uuid} eq '-' ? $source->{node}{uuid} : $source->{node}{received_uuid},
4472 readonly => 1,
4473 TARGET_TYPE => $target_type,
4474 FORCE_PRESERVE => 'preserve forced: created just now',
4475 }, $raw_info);
4476 }
4477
4478 # add info to $config->{SUBVOL_RECEIVED}
4479 $info{received_type} = $target_type || die;
4480 $info{received_subvolume} = $vol_received || die;
4481 $target->{SUBVOL_RECEIVED} //= [];
4482 push(@{$target->{SUBVOL_RECEIVED}}, \%info);
4483 unless($ret) {
4484 $info{ERROR} = 1;
4485 return undef;
4486 }
4487 return 1;
4488 }
4489
4490
4491 # sets $result_vinfo->{CONFIG}->{ABORTED} on failure
4492 # sets $result_vinfo->{SUBVOL_DELETED}
4493 sub macro_delete($$$$;@)
4494 {
4495 my $root_subvol = shift || die;
4496 my $subvol_basename = shift // die;
4497 my $result_vinfo = shift || die;
4498 my $schedule_options = shift || die;
4499 my %delete_options = @_;
4500
4501 my @schedule;
4502 foreach my $vol (@{vinfo_subvol_list($root_subvol, btrbk_direct_leaf => $subvol_basename)}) {
4503 push(@schedule, { value => $vol,
4504 # name => $vol->{PRINT}, # only for logging
4505 btrbk_date => $vol->{node}{BTRBK_DATE},
4506 preserve => $vol->{node}{FORCE_PRESERVE},
4507 });
4508 }
4509 my (undef, $delete) = schedule(
4510 %$schedule_options,
4511 schedule => \@schedule,
4512 preserve_date_in_future => 1,
4513 );
4514
4515 my @delete_success;
4516 foreach my $vol (@$delete) {
4517 # NOTE: we do not abort on qgroup destroy errors
4518 btrfs_qgroup_destroy($vol, %{$delete_options{qgroup}}) if($delete_options{qgroup}->{destroy});
4519 if(btrfs_subvolume_delete($vol, %delete_options)) {
4520 push @delete_success, $vol;
4521 }
4522 }
4523 INFO "Deleted " . scalar(@delete_success) . " subvolumes in: $root_subvol->{PRINT}/$subvol_basename.*";
4524 $result_vinfo->{SUBVOL_DELETED} //= [];
4525 push @{$result_vinfo->{SUBVOL_DELETED}}, @delete_success;
4526
4527 if(scalar(@delete_success) != scalar(@$delete)) {
4528 ABORTED($result_vinfo, "Failed to delete subvolume");
4529 return undef;
4530 }
4531 return 1;
4532 }
4533
4534
4535 sub macro_archive_target($$$;$)
4536 {
4537 my $sroot = shift || die;
4538 my $droot = shift || die;
4539 my $snapshot_name = shift // die;
4540 my $schedule_options = shift // {};
4541 my @schedule;
4542
4543 # NOTE: this is pretty much the same as "resume missing"
4544 my $has_unexpected_location = 0;
4545 foreach my $svol (@{vinfo_subvol_list($sroot, readonly => 1, btrbk_direct_leaf => $snapshot_name, sort => 'path')})
4546 {
4547 if(my $ff = vinfo_match(\@exclude_vf, $svol)) {
4548 INFO "Skipping archive candidate \"$svol->{PRINT}\": Match on exclude pattern \"$ff->{unparsed}\"";
4549 next;
4550 }
4551 next if(get_receive_targets($droot, $svol, exact => 1, warn => 1, ret_unexpected_only => \$has_unexpected_location));
4552 DEBUG "Adding archive candidate: $svol->{PRINT}";
4553
4554 push @schedule, { value => $svol,
4555 btrbk_date => $svol->{node}{BTRBK_DATE},
4556 preserve => $svol->{node}{FORCE_PRESERVE},
4557 };
4558 }
4559
4560 if($has_unexpected_location) {
4561 ABORTED($droot, "Receive targets of archive candidates exist at unexpected location");
4562 WARN "Skipping archiving of \"$sroot->{PRINT}/${snapshot_name}.*\": " . ABORTED_TEXT($droot);
4563 return undef;
4564 }
4565
4566 # add all present archives as informative_only: these are needed for correct results of schedule()
4567 my $last_dvol_date;
4568 foreach my $dvol (@{vinfo_subvol_list($droot, readonly => 1, btrbk_direct_leaf => $snapshot_name)})
4569 {
4570 my $btrbk_date = $dvol->{node}{BTRBK_DATE};
4571 push @schedule, { informative_only => 1,
4572 value => $dvol,
4573 btrbk_date => $btrbk_date,
4574 };
4575
4576 # find last present archive (by btrbk_date, needed for archive_exclude_older below)
4577 $last_dvol_date = $btrbk_date if((not defined($last_dvol_date)) || (cmp_date($btrbk_date, $last_dvol_date) > 0));
4578 }
4579
4580 my ($preserve, undef) = schedule(
4581 schedule => \@schedule,
4582 preserve => config_preserve_hash($droot, "archive"),
4583 preserve_threshold_date => (config_key($droot, "archive_exclude_older") ? $last_dvol_date : undef),
4584 result_preserve_action_text => 'archive',
4585 result_delete_action_text => '',
4586 %$schedule_options
4587 );
4588 my @archive = grep defined, @$preserve; # remove entries with no value from list (archive subvolumes)
4589 my $archive_total = scalar @archive;
4590 my $archive_success = 0;
4591 foreach my $svol (@archive)
4592 {
4593 my ($clone_src, $target_parent_node);
4594 my $parent = get_best_parent($svol, $sroot, $droot,
4595 strict_related => 0,
4596 clone_src => \$clone_src,
4597 target_parent_node => \$target_parent_node);
4598 if(macro_send_receive(source => $svol,
4599 target => $droot,
4600 parent => $parent, # this is <undef> if no suitable parent found
4601 clone_src => $clone_src,
4602 target_parent_node => $target_parent_node,
4603 ))
4604 {
4605 $archive_success++;
4606 }
4607 else {
4608 ERROR("Error while archiving subvolumes, aborting");
4609 last;
4610 }
4611 }
4612
4613 if($archive_total) {
4614 INFO "Archived $archive_success/$archive_total subvolumes";
4615 } else {
4616 INFO "No missing archives found";
4617 }
4618
4619 return $archive_success;
4620 }
4621
4622
4623 sub cmp_date($$)
4624 {
4625 return (($_[0]->[0] <=> $_[1]->[0]) || # unix time
4626 ($_[0]->[1] <=> $_[1]->[1])); # NN
4627 }
4628
4629
4630 sub schedule(@)
4631 {
4632 my %args = @_;
4633 my $schedule = $args{schedule} || die;
4634 my $preserve = $args{preserve} || die;
4635 my $preserve_date_in_future = $args{preserve_date_in_future};
4636 my $preserve_threshold_date = $args{preserve_threshold_date};
4637 my $results_list = $args{results};
4638 my $result_hints = $args{result_hints} // {};
4639 my $result_preserve_action_text = $args{result_preserve_action_text};
4640 my $result_delete_action_text = $args{result_delete_action_text} // 'delete';
4641
4642 my $preserve_day_of_week = $preserve->{dow} || die;
4643 my $preserve_hour_of_day = $preserve->{hod} // die;
4644 my $preserve_min_n = $preserve->{min_n};
4645 my $preserve_min_q = $preserve->{min_q};
4646 my $preserve_hourly = $preserve->{h};
4647 my $preserve_daily = $preserve->{d};
4648 my $preserve_weekly = $preserve->{w};
4649 my $preserve_monthly = $preserve->{m};
4650 my $preserve_yearly = $preserve->{y};
4651
4652 DEBUG "Schedule: " . format_preserve_matrix($preserve, format => "debug_text");
4653
4654 # 0 1 2 3 4 5 6 7 8
4655 # sec, min, hour, mday, mon, year, wday, yday, isdst
4656
4657 # sort the schedule, ascending by date
4658 # regular entries come in front of informative_only
4659 my @sorted_schedule = sort { cmp_date($a->{btrbk_date}, $b->{btrbk_date} ) ||
4660 (($a->{informative_only} ? ($b->{informative_only} ? 0 : 1) : ($b->{informative_only} ? -1 : 0)))
4661 } @$schedule;
4662
4663 DEBUG "Scheduler reference time: " . timestamp(\@tm_now, 'debug-iso');
4664
4665 # first, do our calendar calculations
4666 # - days start on $preserve_hour_of_day (or 00:00 if timestamp_format=short)
4667 # - weeks start on $preserve_day_of_week
4668 # - months start on first $preserve_day_of_week of month
4669 # - years start on first $preserve_day_of_week of year
4670 # NOTE: leap hours are NOT taken into account for $delta_hours
4671 my $now_h = timegm_nocheck( 0, 0, $tm_now[2], $tm_now[3], $tm_now[4], $tm_now[5] ); # use timelocal() here (and below) if you want to honor leap hours
4672
4673 foreach my $href (@sorted_schedule)
4674 {
4675 my @tm = localtime($href->{btrbk_date}->[0]);
4676 my $has_exact_time = $href->{btrbk_date}->[2];
4677 my $delta_hours_from_hod = $tm[2] - ($has_exact_time ? $preserve_hour_of_day : 0);
4678 my $delta_days_from_eow = $tm[6] - $day_of_week_map{$preserve_day_of_week};
4679 if($delta_hours_from_hod < 0) {
4680 $delta_hours_from_hod += 24;
4681 $delta_days_from_eow -= 1;
4682 }
4683 if($delta_days_from_eow < 0) {
4684 $delta_days_from_eow += 7;
4685 }
4686 my $month_corr = $tm[4]; # [0..11]
4687 my $year_corr = $tm[5];
4688 if($tm[3] <= $delta_days_from_eow) {
4689 # our month/year start on first $preserve_day_of_week, corrected value
4690 $month_corr -= 1;
4691 if($month_corr < 0) {
4692 $month_corr = 11;
4693 $year_corr -= 1;
4694 }
4695 }
4696
4697 # check timegm: ignores leap hours
4698 my $delta_hours = int(($now_h - timegm_nocheck( 0, 0, $tm[2], $tm[3], $tm[4], $tm[5] ) ) / (60 * 60));
4699 my $delta_days = int(($delta_hours + $delta_hours_from_hod) / 24); # days from beginning of day
4700 my $delta_weeks = int(($delta_days + $delta_days_from_eow) / 7); # weeks from beginning of week
4701 my $delta_years = ($tm_now[5] - $year_corr);
4702 my $delta_months = $delta_years * 12 + ($tm_now[4] - $month_corr);
4703
4704 $href->{delta_hours} = $delta_hours;
4705 $href->{delta_days} = $delta_days;
4706 $href->{delta_weeks} = $delta_weeks;
4707 $href->{delta_months} = $delta_months;
4708 $href->{delta_years} = $delta_years;
4709
4710 # these are only needed for text output (format_preserve_delta)
4711 $href->{year} = $year_corr + 1900;
4712 $href->{month} = $month_corr + 1;
4713 $href->{delta_hours_from_hod} = $delta_hours_from_hod;
4714 $href->{delta_days_from_eow} = $delta_days_from_eow;
4715 $href->{real_hod} = $preserve_hour_of_day if($has_exact_time);
4716
4717 if($preserve_date_in_future && ($delta_hours < 0)) {
4718 $href->{preserve} = "preserve forced: " . -($delta_hours) . " hours in the future";
4719 }
4720 }
4721
4722 my %first_in_delta_hours;
4723 my %first_in_delta_days;
4724 my %first_in_delta_weeks;
4725 my %first_weekly_in_delta_months;
4726 my %first_monthly_in_delta_years;
4727
4728 # filter "preserve all within N days/weeks/..."
4729 foreach my $href (@sorted_schedule) {
4730 if($preserve_min_q) {
4731 if($preserve_min_q eq 'all') {
4732 $href->{preserve} = "preserve min: all";
4733 } elsif($preserve_min_q eq 'h') {
4734 $href->{preserve} = "preserve min: $href->{delta_hours} hours ago" if($href->{delta_hours} <= $preserve_min_n);
4735 } elsif($preserve_min_q eq 'd') {
4736 $href->{preserve} = "preserve min: $href->{delta_days} days ago" if($href->{delta_days} <= $preserve_min_n);
4737 } elsif($preserve_min_q eq 'w') {
4738 $href->{preserve} = "preserve min: $href->{delta_weeks} weeks ago" if($href->{delta_weeks} <= $preserve_min_n);
4739 } elsif($preserve_min_q eq 'm') {
4740 $href->{preserve} = "preserve min: $href->{delta_months} months ago" if($href->{delta_months} <= $preserve_min_n);
4741 } elsif($preserve_min_q eq 'y') {
4742 $href->{preserve} = "preserve min: $href->{delta_years} years ago" if($href->{delta_years} <= $preserve_min_n);
4743 }
4744 }
4745 $first_in_delta_hours{$href->{delta_hours}} //= $href;
4746 }
4747 if($preserve_min_q && ($preserve_min_q eq 'latest') && (scalar @sorted_schedule)) {
4748 my $href = $sorted_schedule[-1];
4749 $href->{preserve} = 'preserve min: latest';
4750 }
4751
4752 # filter hourly, daily, weekly, monthly, yearly
4753 foreach (sort {$b <=> $a} keys %first_in_delta_hours) {
4754 my $href = $first_in_delta_hours{$_} || die;
4755 if($preserve_hourly && (($preserve_hourly eq 'all') || ($href->{delta_hours} <= $preserve_hourly))) {
4756 $href->{preserve} = "preserve hourly: first of hour, $href->{delta_hours} hours ago";
4757 }
4758 $first_in_delta_days{$href->{delta_days}} //= $href;
4759 }
4760 foreach (sort {$b <=> $a} keys %first_in_delta_days) {
4761 my $href = $first_in_delta_days{$_} || die;
4762 if($preserve_daily && (($preserve_daily eq 'all') || ($href->{delta_days} <= $preserve_daily))) {
4763 $href->{preserve} = "preserve daily: first of day" . ($href->{real_hod} ? sprintf(" (starting at %02u:00)", $href->{real_hod}) : "") . ", $href->{delta_days} days ago"
4764 . (defined($href->{real_hod}) ? ($href->{delta_hours_from_hod} ? ", $href->{delta_hours_from_hod}h after " : ", at ") . sprintf("%02u:00", $href->{real_hod}) : "");
4765 }
4766 $first_in_delta_weeks{$href->{delta_weeks}} //= $href;
4767 }
4768 foreach (sort {$b <=> $a} keys %first_in_delta_weeks) {
4769 my $href = $first_in_delta_weeks{$_} || die;
4770 if($preserve_weekly && (($preserve_weekly eq 'all') || ($href->{delta_weeks} <= $preserve_weekly))) {
4771 $href->{preserve} = "preserve weekly: $href->{delta_weeks} weeks ago," . _format_preserve_delta($href, $preserve_day_of_week);
4772 }
4773 $first_weekly_in_delta_months{$href->{delta_months}} //= $href;
4774 }
4775 foreach (sort {$b <=> $a} keys %first_weekly_in_delta_months) {
4776 my $href = $first_weekly_in_delta_months{$_} || die;
4777 if($preserve_monthly && (($preserve_monthly eq 'all') || ($href->{delta_months} <= $preserve_monthly))) {
4778 $href->{preserve} = "preserve monthly: first weekly of month $href->{year}-" . sprintf("%02u", $href->{month}) . " ($href->{delta_months} months ago," . _format_preserve_delta($href, $preserve_day_of_week) . ")";
4779 }
4780 $first_monthly_in_delta_years{$href->{delta_years}} //= $href;
4781 }
4782 foreach (sort {$b <=> $a} keys %first_monthly_in_delta_years) {
4783 my $href = $first_monthly_in_delta_years{$_} || die;
4784 if($preserve_yearly && (($preserve_yearly eq 'all') || ($href->{delta_years} <= $preserve_yearly))) {
4785 $href->{preserve} = "preserve yearly: first weekly of year $href->{year} ($href->{delta_years} years ago," . _format_preserve_delta($href, $preserve_day_of_week) . ")";
4786 }
4787 }
4788
4789 # assemble results
4790 my @delete;
4791 my @preserve;
4792 my %result_base = ( %$preserve,
4793 scheme => format_preserve_matrix($preserve),
4794 %$result_hints,
4795 );
4796 my $count_defined = 0;
4797 foreach my $href (@sorted_schedule)
4798 {
4799 my $result_reason_text = $href->{preserve};
4800 my $result_action_text;
4801
4802 unless($href->{informative_only}) {
4803 if($href->{preserve}) {
4804 if($preserve_threshold_date && (cmp_date($href->{btrbk_date}, $preserve_threshold_date) <= 0)) {
4805 # older than threshold, do not add to preserve list
4806 $result_reason_text = "$result_reason_text, ignored (archive_exclude_older) older than existing archive";
4807 }
4808 else {
4809 push(@preserve, $href->{value});
4810 $result_action_text = $result_preserve_action_text;
4811 }
4812 }
4813 else {
4814 push(@delete, $href->{value});
4815 $result_action_text = $result_delete_action_text;
4816 }
4817 $count_defined++;
4818 }
4819
4820 TRACE join(" ", "schedule: $href->{value}{PRINT}", ($href->{informative_only} ? "(informative_only)" : uc($result_action_text || "-")), ($result_reason_text // "-")) if($do_trace && $href->{value} && $href->{value}{PRINT});
4821 push @$results_list, { %result_base,
4822 action => $result_action_text,
4823 reason => $result_reason_text,
4824 value => $href->{value},
4825 } if($results_list);
4826 }
4827 DEBUG "Preserving " . @preserve . "/" . $count_defined . " items";
4828 return (\@preserve, \@delete);
4829 }
4830
4831
4832 sub _format_preserve_delta($$$)
4833 {
4834 my $href = shift;
4835 my $preserve_day_of_week = shift;
4836 my $s = "";
4837 $s .= " $href->{delta_days_from_eow}d" if($href->{delta_days_from_eow});
4838 $s .= " $href->{delta_hours_from_hod}h" if($href->{delta_hours_from_hod});
4839 return ($s ? "$s after " : " at ") . $preserve_day_of_week . (defined($href->{real_hod}) ? sprintf(" %02u:00", $href->{real_hod}) : "");
4840 }
4841
4842
4843 sub format_preserve_matrix($@)
4844 {
4845 my $preserve = shift || die;
4846 my %opts = @_;
4847 my $format = $opts{format} // "short";
4848
4849 if($format eq "debug_text") {
4850 my @out;
4851 my %trans = ( h => 'hours', d => 'days', w => 'weeks', m => 'months', y => 'years' );
4852 if($preserve->{min_q} && ($preserve->{min_q} eq 'all')) {
4853 push @out, "all forever";
4854 }
4855 else {
4856 push @out, "latest" if($preserve->{min_q} && ($preserve->{min_q} eq 'latest'));
4857 push @out, "all within $preserve->{min_n} $trans{$preserve->{min_q}}" if($preserve->{min_n} && $preserve->{min_q});
4858 push @out, "first of day (starting at " . sprintf("%02u:00", $preserve->{hod}) . ") for $preserve->{d} days" if($preserve->{d});
4859 unless($preserve->{d} && ($preserve->{d} eq 'all')) {
4860 push @out, "first daily in week (starting on $preserve->{dow}) for $preserve->{w} weeks" if($preserve->{w});
4861 unless($preserve->{w} && ($preserve->{w} eq 'all')) {
4862 push @out, "first weekly of month for $preserve->{m} months" if($preserve->{m});
4863 unless($preserve->{m} && ($preserve->{m} eq 'all')) {
4864 push @out, "first weekly of year for $preserve->{y} years" if($preserve->{y});
4865 }
4866 }
4867 }
4868 }
4869 return 'preserving ' . join('; ', @out);
4870 }
4871
4872 my $s = "";
4873 if($preserve->{min_q} && ($preserve->{min_q} eq 'all')) {
4874 $s = '*d+';
4875 }
4876 else {
4877 # $s .= '.+' if($preserve->{min_q} && ($preserve->{min_q} eq 'latest'));
4878 $s .= $preserve->{min_n} . $preserve->{min_q} . '+' if($preserve->{min_n} && $preserve->{min_q});
4879 foreach (qw(h d w m y)) {
4880 my $val = $preserve->{$_} // 0;
4881 next unless($val);
4882 $val = '*' if($val eq 'all');
4883 $s .= ($s ? ' ' : '') . $val . $_;
4884 }
4885 if(($format ne "config") && ($preserve->{d} || $preserve->{w} || $preserve->{m} || $preserve->{y})) {
4886 $s .= " ($preserve->{dow}, " . sprintf("%02u:00", $preserve->{hod}) . ")";
4887 }
4888 }
4889 return $s;
4890 }
4891
4892
4893 sub timestamp($$;$)
4894 {
4895 my $time = shift // die; # unixtime, or arrayref from localtime()
4896 my $format = shift;
4897 my $tm_is_utc = shift;
4898 my @tm = ref($time) ? @$time : localtime($time);
4899 my $ts;
4900 # NOTE: can't use POSIX::strftime(), as "%z" always prints offset of local timezone!
4901
4902 if($format eq "short") {
4903 return sprintf('%04u%02u%02u', $tm[5] + 1900, $tm[4] + 1, $tm[3]);
4904 }
4905 elsif($format eq "long") {
4906 return sprintf('%04u%02u%02uT%02u%02u', $tm[5] + 1900, $tm[4] + 1, $tm[3], $tm[2], $tm[1]);
4907 }
4908 elsif($format eq "long-iso") {
4909 $ts = sprintf('%04u%02u%02uT%02u%02u%02u', $tm[5] + 1900, $tm[4] + 1, $tm[3], $tm[2], $tm[1], $tm[0]);
4910 }
4911 elsif($format eq "debug-iso") {
4912 $ts = sprintf('%04u-%02u-%02uT%02u:%02u:%02u', $tm[5] + 1900, $tm[4] + 1, $tm[3], $tm[2], $tm[1], $tm[0]);
4913 }
4914 else { die; }
4915
4916 if($tm_is_utc) {
4917 $ts .= '+0000'; # or 'Z'
4918 } else {
4919 my $offset = timegm(@tm) - timelocal(@tm);
4920 if($offset < 0) { $ts .= '-'; $offset = -$offset; } else { $ts .= '+'; }
4921 my $h = int($offset / (60 * 60));
4922 die if($h > 24); # sanity check, something went really wrong
4923 $ts .= sprintf('%02u%02u', $h, int($offset / 60) % 60);
4924 }
4925 return $ts;
4926
4927 return undef;
4928 }
4929
4930
4931 sub print_header(@)
4932 {
4933 my %args = @_;
4934 my $config = $args{config};
4935
4936 print "--------------------------------------------------------------------------------\n";
4937 print "$args{title} ($VERSION_INFO)\n\n";
4938 if($args{time}) {
4939 print " Date: " . localtime($args{time}) . "\n";
4940 }
4941 if($config) {
4942 print " Config: " . config_key($config, "SRC_FILE") . "\n";
4943 }
4944 if($dryrun) {
4945 print " Dryrun: YES\n";
4946 }
4947 if($config && $config->{CMDLINE_FILTER_LIST}) {
4948 my @list = @{$config->{CMDLINE_FILTER_LIST}};
4949 print " Filter: ";
4950 print join("\n ", @list);
4951 print "\n";
4952 }
4953 if($args{info}) {
4954 print "\n" . join("\n", grep(defined, @{$args{info}})) . "\n";
4955 }
4956 if($args{options} && (scalar @{$args{options}})) {
4957 print "\nOptions:\n ";
4958 print join("\n ", @{$args{options}});
4959 print "\n";
4960 }
4961 if($args{legend}) {
4962 print "\nLegend:\n ";
4963 print join("\n ", @{$args{legend}});
4964 print "\n";
4965 }
4966 print "--------------------------------------------------------------------------------\n";
4967 print "\n" if($args{paragraph});
4968 }
4969
4970
4971 sub print_footer($$)
4972 {
4973 my $config = shift;
4974 my $exit_status = shift;
4975 if($exit_status) {
4976 print "\nNOTE: Some errors occurred, which may result in missing backups!\n";
4977 print "Please check warning and error messages above.\n";
4978 my @fix_manually_text = _config_collect_values($config, "FIX_MANUALLY");
4979 if(scalar(@fix_manually_text)) {
4980 my @unique = do { my %seen; grep { !$seen{$_}++ } @fix_manually_text };
4981 print join("\n", @unique) . "\n";
4982 }
4983 }
4984 if($dryrun) {
4985 print "\nNOTE: Dryrun was active, none of the operations above were actually executed!\n";
4986 }
4987 }
4988
4989
4990 sub print_table($;$)
4991 {
4992 my $data = shift;
4993 my $spacing = shift // " ";
4994 my $maxlen = 0;
4995 foreach (@$data) {
4996 $maxlen = length($_->[0]) if($maxlen < length($_->[0]));
4997 }
4998 foreach (@$data) {
4999 print $_->[0] . ((' ' x ($maxlen - length($_->[0]))) . $spacing) . $_->[1] . "\n";
5000 }
5001 }
5002
5003
5004 sub print_formatted(@)
5005 {
5006 my $format_key = shift || die;
5007 my $data = shift || die;
5008 my $default_format = "table";
5009 my %args = @_;
5010 my $title = $args{title};
5011 my $table_format = ref($format_key) ? $format_key : $table_formats{$format_key};
5012 my $format = $args{output_format} || $output_format || $default_format;
5013 my $pretty = $args{pretty} // $output_pretty;
5014 my $no_header = $args{no_header};
5015 my $fh = $args{outfile} // *STDOUT;
5016 my $table_spacing = 2;
5017 my $empty_cell_char = $args{empty_cell_char} // "-";
5018
5019 my @keys;
5020 my %ralign;
5021 my %hide_column;
5022 if($format =~ s/^col:\s*(h:)?\s*//) {
5023 $no_header = 1 if($1);
5024 foreach (split(/\s*,\s*/, $format)) {
5025 $ralign{$_} = 1 if s/:R(ALIGN)?$//i;
5026 push @keys, lc($_);
5027 }
5028 }
5029 else {
5030 unless(exists($table_format->{$format})) {
5031 WARN "Unsupported output format \"$format\", defaulting to \"$default_format\" format.";
5032 $format = $default_format;
5033 }
5034 @keys = @{$table_format->{$format}};
5035 %ralign = %{$table_format->{RALIGN} // {}};
5036 }
5037 # strips leading "-" from @keys
5038 %hide_column = map { $_ => 1 } grep { s/^-// } @keys;
5039
5040 if($format eq "single_column")
5041 {
5042 # single-column: newline separated values, no headers
5043 my $key = $keys[0];
5044 foreach (grep defined, map $_->{$key}, @$data) {
5045 print $fh $_ . "\n" if($_ ne "");
5046 }
5047 }
5048 elsif($format eq "raw")
5049 {
5050 # output: key0="value0" key1="value1" ...
5051 foreach my $row (@$data) {
5052 print $fh "format=\"$format_key\" ";
5053 print $fh join(' ', map { "$_=" . quoteshell(($row->{$_} // "")) } @keys) . "\n";
5054 }
5055 }
5056 elsif(($format eq "tlog") || ($format eq "syslog"))
5057 {
5058 # output: value0 value1, ...
5059 unless($no_header) {
5060 print $fh join(' ', map uc($_), @keys) . "\n"; # unaligned upper case headings
5061 }
5062 foreach my $row (@$data) {
5063 my $line = join(' ', map { ((defined($row->{$_}) && ($_ eq "message")) ? '# ' : '') . ($row->{$_} // "-") } @keys);
5064 if($format eq "syslog") { # dirty hack, ignore outfile on syslog format
5065 syslog($line);
5066 } else {
5067 print $fh ($line . "\n");
5068 }
5069 }
5070 }
5071 else
5072 {
5073 # Text::CharWidth does it correctly with wide chars (e.g. asian) taking up two columns
5074 my $termwidth = eval_quiet { require Text::CharWidth; } ? \&Text::CharWidth::mbswidth :
5075 eval_quiet { require Encode; } ? sub { length(Encode::decode_utf8(shift)) } :
5076 sub { length(shift) };
5077
5078 # sanitize and calculate maxlen for each column
5079 my %maxlen = map { $_ => $no_header ? 0 : length($_) } @keys;
5080 my @formatted_data;
5081 foreach my $row (@$data) {
5082 my %formatted_row;
5083 foreach my $key (@keys) {
5084 my $val = $row->{$key};
5085 $val = join(',', @$val) if(ref $val eq "ARRAY");
5086
5087 $hide_column{$key} = 0 if(defined($val));
5088 $val = $empty_cell_char if(!defined($val) || ($val eq ""));
5089 $formatted_row{$key} = $val;
5090 my $vl = $termwidth->($val);
5091 $maxlen{$key} = $vl if($maxlen{$key} < $vl);
5092 }
5093 push @formatted_data, \%formatted_row;
5094 }
5095 my @visible_keys = grep !$hide_column{$_}, @keys;
5096
5097 # print title
5098 if($title) {
5099 print $fh "$title\n";
5100 print $fh '-' x length($title) . "\n"; # separator line
5101 }
5102
5103 # print keys (headings)
5104 unless($no_header) {
5105 my $fill = 0;
5106 foreach (@visible_keys) {
5107 print $fh ' ' x $fill;
5108 $fill = $maxlen{$_} - length($_);
5109 if($pretty) {
5110 # use aligned lower case headings (with separator line below)
5111 if($ralign{$_}) {
5112 print $fh ' ' x $fill;
5113 $fill = 0;
5114 }
5115 print $fh $_;
5116 } else {
5117 print $fh uc($_); # default unaligned upper case headings
5118 }
5119 $fill += $table_spacing;
5120 }
5121 print $fh "\n";
5122
5123 $fill = 0;
5124 if($pretty) { # separator line after header
5125 foreach (@visible_keys) {
5126 print $fh ' ' x $fill;
5127 print $fh '-' x $maxlen{$_};
5128 $fill = $table_spacing;
5129 }
5130 print $fh "\n";
5131 # alternative (all above in one line ;)
5132 #print $fh join(' ' x $table_spacing, map { '-' x ($maxlen{$_}) } @keys) . "\n";
5133 }
5134 }
5135
5136 # print values
5137 foreach my $row (@formatted_data) {
5138 my $fill = 0;
5139 foreach (@visible_keys) {
5140 my $val = $row->{$_};
5141 print $fh ' ' x $fill;
5142 $fill = $maxlen{$_} - $termwidth->($val);
5143 if($ralign{$_}) {
5144 print $fh ' ' x $fill;
5145 $fill = 0;
5146 }
5147 print $fh $val;
5148 $fill += $table_spacing;
5149 }
5150 print $fh "\n";
5151 }
5152
5153 # print additional newline for paragraphs
5154 if($args{paragraph}) {
5155 print $fh "\n";
5156 }
5157 }
5158 }
5159
5160
5161 sub print_size($)
5162 {
5163 my $size = shift;
5164 if($output_format && ($output_format eq "raw")) {
5165 return $size;
5166 }
5167 return "-" if($size == 0);
5168 my ($unit, $mul);
5169 if(@output_unit) {
5170 ($unit, $mul) = @output_unit;
5171 }
5172 else {
5173 ($unit, $mul) = ("KiB", 1024);
5174 ($unit, $mul) = ("MiB", $mul * 1024) if($size > $mul * 1024);
5175 ($unit, $mul) = ("GiB", $mul * 1024) if($size > $mul * 1024);
5176 ($unit, $mul) = ("TiB", $mul * 1024) if($size > $mul * 1024);
5177 }
5178 return $size if($mul == 1);
5179 return sprintf('%.2f', ($size / $mul)) . " $unit";
5180 }
5181
5182
5183 sub _origin_tree
5184 {
5185 my $prefix = shift;
5186 my $node = shift // die;
5187 my $lines = shift;
5188 my $nodelist = shift;
5189 my $depth = shift // 0;
5190 my $seen = shift // [];
5191 my $norecurse = shift;
5192 my $uuid = $node->{uuid} || die;
5193
5194 # cache a bit, this might be large
5195 # note: root subvolumes dont have REL_PATH
5196 $nodelist //= [ (sort { ($a->{REL_PATH} // "") cmp ($b->{REL_PATH} // "") } values %uuid_cache) ];
5197
5198 my $prefix_spaces = ' ' x (($depth * 4) - ($prefix ? 4 : 0));
5199 push(@$lines, { tree => "${prefix_spaces}${prefix}" . _fs_path($node),
5200 uuid => $node->{uuid},
5201 parent_uuid => $node->{parent_uuid},
5202 received_uuid => $node->{received_uuid},
5203 });
5204
5205 # handle deep recursion
5206 return 0 if(grep /^$uuid$/, @$seen);
5207
5208 if($node->{parent_uuid} ne '-') {
5209 my $parent_node = $uuid_cache{$node->{parent_uuid}};
5210 if($parent_node) {
5211 if($norecurse) {
5212 push(@$lines,{ tree => "${prefix_spaces} ^-- ...",
5213 uuid => $parent_node->{uuid},
5214 parent_uuid => $parent_node->{parent_uuid},
5215 received_uuid => $parent_node->{received_uuid},
5216 recursion => 'stop_recursion',
5217 });
5218 return 0;
5219 }
5220 if($parent_node->{readonly}) {
5221 _origin_tree("^-- ", $parent_node, $lines, $nodelist, $depth + 1, undef, 1); # end recursion
5222 }
5223 else {
5224 _origin_tree("^-- ", $parent_node, $lines, $nodelist, $depth + 1);
5225 }
5226 }
5227 else {
5228 push(@$lines,{ tree => "${prefix_spaces} ^-- <unknown uuid=$node->{parent_uuid}>" });
5229 }
5230 }
5231
5232 return 0 if($norecurse);
5233 push(@$seen, $uuid);
5234
5235 if($node->{received_uuid} ne '-') {
5236 my $received_uuid = $node->{received_uuid};
5237 my @receive_parents; # there should be only one!
5238 my @receive_twins;
5239
5240 foreach (@$nodelist) {
5241 next if($_->{uuid} eq $uuid);
5242 if($received_uuid eq $_->{uuid} && $_->{readonly}) {
5243 _origin_tree("", $_, \@receive_parents, $nodelist, $depth, $seen);
5244 }
5245 elsif(($_->{received_uuid} ne '-') && ($received_uuid eq $_->{received_uuid}) && $_->{readonly}) {
5246 _origin_tree("", $_, \@receive_twins, $nodelist, $depth, $seen, 1); # end recursion
5247 }
5248 }
5249 push @$lines, @receive_twins;
5250 push @$lines, @receive_parents;
5251 }
5252
5253 return 0;
5254 }
5255
5256
5257 sub exit_status
5258 {
5259 my $config = shift;
5260 foreach my $subsection (@{$config->{SUBSECTION}}) {
5261 return 10 if(IS_ABORTED($subsection, "abort_"));
5262 return 10 if(defined($subsection->{FIX_MANUALLY})); # treated as errors
5263 return 10 if(exit_status($subsection));
5264 }
5265 return 0;
5266 }
5267
5268
5269
5270 MAIN:
5271 {
5272 # NOTE: Since v0.26.0, btrbk does not enable taint mode (perl -T) by
5273 # default, and does not hardcode $PATH anymore.
5274 #
5275 # btrbk still does all taint checks, and can be run in taint mode.
5276 # In order to enable taint mode, run `perl -T btrbk`.
5277 #
5278 # see: perlrun(1), perlsec(1)
5279 #
5280 my $taint_mode_enabled = eval '${^TAINT}';
5281 if($taint_mode_enabled) {
5282 # we are running in tainted mode (perl -T), sanitize %ENV
5283 delete @ENV{qw(IFS CDPATH ENV BASH_ENV)};
5284
5285 # in taint mode, perl needs an untainted $PATH.
5286 $ENV{PATH} = '/sbin:/bin:/usr/sbin:/usr/bin';
5287 }
5288
5289 Getopt::Long::Configure qw(gnu_getopt);
5290 my $start_time = time;
5291 @tm_now = localtime($start_time);
5292
5293 my @config_override_cmdline;
5294 my @exclude_cmdline;
5295 my ($config_cmdline, $lockfile_cmdline, $print_schedule,
5296 $preserve_snapshots, $preserve_backups, $wipe_snapshots, $skip_snapshots, $skip_backups,
5297 $archive_raw, $extents_related,
5298 );
5299
5300 # Calling btrbk via "lsbtr" symlink acts as an alias for "btrbk ls",
5301 # while also changing the semantics of the command line options.
5302 $program_name = $0;
5303 $program_name =~ s/^.*\///; # remove path
5304 my @getopt_options = (
5305 # common options
5306 'help|h' => sub { HELP_MESSAGE; exit 0; },
5307 'version' => sub { VERSION_MESSAGE; exit 0; },
5308 'quiet|q' => \$quiet,
5309 'verbose|v' => sub { $loglevel = ($loglevel =~ /^[0-9]+$/) ? $loglevel+1 : 2; },
5310 'loglevel|l=s' => \$loglevel,
5311 'format=s' => \$output_format,
5312 'single-column|1' => sub { $output_format = "single_column" },
5313 'pretty' => \$output_pretty,
5314 'config|c=s' => \$config_cmdline,
5315 'override=s' => \@config_override_cmdline, # e.g. --override=incremental=no
5316 'lockfile=s' => \$lockfile_cmdline,
5317 );
5318 push @getopt_options, ($program_name eq "lsbtr") ? (
5319 # "lsbtr" options
5320 'long|l' => sub { $output_format = "table" },
5321 'uuid|u' => sub { $output_format = "long" },
5322 'raw' => sub { $output_format = "raw" },
5323 ) : (
5324 # "btrbk" options
5325 'dry-run|n' => \$dryrun,
5326 'exclude=s' => \@exclude_cmdline,
5327 'preserve|p' => sub { $preserve_snapshots = "preserve", $preserve_backups = "preserve" },
5328 'preserve-snapshots' => sub { $preserve_snapshots = "preserve-snapshots" },
5329 'preserve-backups' => sub { $preserve_backups = "preserve-backups" },
5330 'wipe' => \$wipe_snapshots,
5331 'progress' => \$show_progress,
5332 'related' => \$extents_related,
5333 'table|t' => sub { $output_format = "table" },
5334 'long|L' => sub { $output_format = "long" },
5335 'print-schedule|S' => \$print_schedule,
5336 'raw' => \$archive_raw,
5337 'bytes' => sub { @output_unit = ("", 1 ) },
5338 'kbytes' => sub { @output_unit = ("KiB", 1024 ) },
5339 'mbytes' => sub { @output_unit = ("MiB", 1024 * 1024 ) },
5340 'gbytes' => sub { @output_unit = ("GiB", 1024 * 1024 * 1024 ) },
5341 'tbytes' => sub { @output_unit = ("TiB", 1024 * 1024 * 1024 * 1024 ) },
5342 );
5343 unless(GetOptions(@getopt_options)) {
5344 ERROR_HELP_MESSAGE;
5345 exit 2;
5346 }
5347 if($program_name eq "lsbtr") {
5348 unshift @ARGV, './' unless(@ARGV); # default to current path
5349 unshift @ARGV, "ls"; # implicit "btrbk ls"
5350 }
5351 my $command = shift @ARGV;
5352 unless($command) {
5353 HELP_MESSAGE;
5354 exit 2;
5355 }
5356
5357 # assign command line options
5358 @config_src = ( $config_cmdline ) if($config_cmdline);
5359 $loglevel = { error => 0, warn => 1, warning => 1, info => 2, debug => 3, trace => 4 }->{$loglevel} // $loglevel;
5360 unless($loglevel =~ /^[0-9]+$/) {
5361 ERROR "Unknown loglevel: $loglevel";
5362 ERROR_HELP_MESSAGE;
5363 exit 2;
5364 }
5365 $do_trace = 1 if($loglevel >= 4);
5366 require_data_dumper() if($do_trace || ($VERSION =~ /-dev$/));
5367
5368 # check command line options
5369 if($show_progress && (not check_exe('mbuffer'))) {
5370 WARN 'Found option "--progress", but required executable "mbuffer" does not exist on your system. Please install "mbuffer".';
5371 $show_progress = 0;
5372 }
5373 my ($action_run, $action_usage, $action_resolve, $action_diff, $action_extents, $action_origin, $action_config_print, $action_list, $action_clean, $action_archive, $action_ls);
5374 my @filter_args;
5375 my @subvol_args;
5376 my $args_expected_min = 0;
5377 my $args_expected_max = 9999;
5378 my $fallback_default_config;
5379 my $subvol_args_allow_relative;
5380 my $subvol_args_init;
5381 if(($command eq "run") || ($command eq "dryrun")) {
5382 $action_run = 1;
5383 $dryrun = 1 if($command eq "dryrun");
5384 @filter_args = @ARGV;
5385 }
5386 elsif($command eq "snapshot") {
5387 $action_run = 1;
5388 $skip_backups = "snapshot";
5389 $preserve_backups = "snapshot";
5390 @filter_args = @ARGV;
5391 }
5392 elsif($command eq "resume") {
5393 $action_run = 1;
5394 $skip_snapshots = "resume";
5395 @filter_args = @ARGV;
5396 }
5397 elsif($command eq "prune") {
5398 $action_run = 1;
5399 $skip_snapshots = "prune";
5400 $skip_backups = "prune";
5401 @filter_args = @ARGV;
5402 }
5403 elsif ($command eq "clean") {
5404 $action_clean = 1;
5405 @filter_args = @ARGV;
5406 }
5407 elsif ($command eq "archive") {
5408 $action_archive = 1;
5409 $fallback_default_config = 1;
5410 $args_expected_min = $args_expected_max = 2;
5411 $subvol_args_allow_relative = 1;
5412 @subvol_args = @ARGV;
5413 }
5414 elsif ($command eq "usage") {
5415 $action_usage = 1;
5416 @filter_args = @ARGV;
5417 }
5418 elsif ($command eq "ls") {
5419 $action_ls = 1;
5420 $fallback_default_config = 1;
5421 $args_expected_min = 1;
5422 $subvol_args_allow_relative = 1;
5423 @subvol_args = @ARGV;
5424 }
5425 elsif ($command eq "diff") {
5426 $action_diff = 1;
5427 $fallback_default_config = 1;
5428 $args_expected_min = $args_expected_max = 2;
5429 $subvol_args_init = "restrict_same_fs deny_root_subvol";
5430 $subvol_args_allow_relative = 1;
5431 @subvol_args = @ARGV;
5432 }
5433 elsif ($command eq "extents") {
5434 my $subcommand = shift @ARGV // "";
5435 if(($subcommand eq "list") ||
5436 ($subcommand eq "diff")) {
5437 $action_extents = $subcommand;
5438 }
5439 else { # defaults to "list"
5440 unshift @ARGV, $subcommand;
5441 $action_extents = "list";
5442 }
5443 $fallback_default_config = 1;
5444 $args_expected_min = 1;
5445 $subvol_args_init = "restrict_same_fs";
5446 $subvol_args_allow_relative = 1;
5447 my $excl;
5448 foreach(@ARGV) {
5449 # subvol_arg... "exclusive" filter_arg...
5450 if($_ eq "exclusive") {
5451 $excl = 1;
5452 } else {
5453 push @subvol_args, $_;
5454 push @filter_args, $_ if($excl);
5455 }
5456 }
5457 }
5458 elsif ($command eq "origin") {
5459 $action_origin = 1;
5460 $args_expected_min = $args_expected_max = 1;
5461 $subvol_args_init = "deny_root_subvol";
5462 $subvol_args_allow_relative = 1;
5463 @subvol_args = @ARGV;
5464 }
5465 elsif($command eq "list") {
5466 my $subcommand = shift @ARGV // "";
5467 if(($subcommand eq "config") ||
5468 ($subcommand eq "volume") ||
5469 ($subcommand eq "source") ||
5470 ($subcommand eq "target"))
5471 {
5472 $action_list = $subcommand;
5473 }
5474 elsif(($subcommand eq "all") ||
5475 ($subcommand eq "snapshots") ||
5476 ($subcommand eq "backups") ||
5477 ($subcommand eq "latest"))
5478 {
5479 $action_resolve = $subcommand;
5480 }
5481 else {
5482 $action_resolve = "all";
5483 unshift @ARGV, $subcommand if($subcommand ne "");
5484 }
5485 @filter_args = @ARGV;
5486 }
5487 elsif($command eq "stats") {
5488 $action_resolve = "stats";
5489 @filter_args = @ARGV;
5490 }
5491 elsif ($command eq "config") {
5492 my $subcommand = shift @ARGV // "";
5493 @filter_args = @ARGV;
5494 if(($subcommand eq "print") || ($subcommand eq "print-all")) {
5495 $action_config_print = $subcommand;
5496 }
5497 elsif($subcommand eq "list") {
5498 $action_list = "config";
5499 }
5500 else {
5501 ERROR "Unknown subcommand for \"config\" command: $subcommand";
5502 ERROR_HELP_MESSAGE;
5503 exit 2;
5504 }
5505 }
5506 else {
5507 ERROR "Unrecognized command: $command";
5508 ERROR_HELP_MESSAGE;
5509 exit 2;
5510 }
5511 if(($args_expected_min > scalar(@ARGV)) || ($args_expected_max < scalar(@ARGV))) {
5512 ERROR "Incorrect number of arguments";
5513 ERROR_HELP_MESSAGE;
5514 exit 2;
5515 }
5516
5517 # input validation
5518 foreach (@subvol_args) {
5519 my ($url_prefix, $path) = check_url($_);
5520 if(!defined($path) && $subvol_args_allow_relative && ($url_prefix eq "") && (-d $_)) {
5521 $path = check_file(abs_path($_), { absolute => 1, sanitize => 1 });
5522 }
5523 unless(defined($path)) {
5524 ERROR "Bad argument: not a subvolume declaration: $_";
5525 ERROR_HELP_MESSAGE;
5526 exit 2;
5527 }
5528 $_ = $url_prefix . $path;
5529 }
5530 my @filter_vf;
5531 foreach (@filter_args) {
5532 my $vf = vinfo_filter_statement($_);
5533 unless($vf) {
5534 ERROR "Bad argument: invalid filter statement: $_";
5535 ERROR_HELP_MESSAGE;
5536 exit 2;
5537 }
5538 push @filter_vf, $vf;
5539 }
5540 foreach (@exclude_cmdline) {
5541 my $vf = vinfo_filter_statement($_);
5542 unless($vf) {
5543 ERROR "Bad argument: invalid filter statement: --exclude='$_'";
5544 ERROR_HELP_MESSAGE;
5545 exit 2;
5546 }
5547 push @exclude_vf, $vf;
5548 }
5549 foreach(@config_override_cmdline) {
5550 if(/(.*?)=(.*)/) {
5551 my $key = $1;
5552 my $value = $2;
5553 unless(append_config_option(\%config_override, $key, $value, "OVERRIDE", error_statement => "in option \"--override\"")) {
5554 ERROR_HELP_MESSAGE;
5555 exit 2;
5556 }
5557 }
5558 else {
5559 ERROR "Option \"override\" requires \"<config_option>=<value>\" format";
5560 ERROR_HELP_MESSAGE;
5561 exit 2;
5562 }
5563 }
5564 if(defined($lockfile_cmdline)) {
5565 unless($lockfile = check_file($lockfile_cmdline, { absolute => 1, relative => 1 },
5566 error_statement => 'for option --lockfile')) {
5567 exit 2;
5568 }
5569 }
5570
5571
5572 INFO "$VERSION_INFO (" . localtime($start_time) . ")";
5573 action("startup", status => "v$VERSION", message => $VERSION_INFO, time => $start_time);
5574
5575
5576 #
5577 # parse config file
5578 #
5579 my $config;
5580 if(my $config_file = _config_file(@config_src)) {
5581 INFO "Using configuration: $config_file";
5582 $config = parse_config($config_file);
5583 exit 2 unless($config);
5584 }
5585 elsif($fallback_default_config) {
5586 INFO "Configuration file not found, falling back to defaults";
5587 $config = init_config();
5588 }
5589 else {
5590 ERROR "Configuration file not found: " . join(', ', @config_src);
5591 exit 2;
5592 }
5593 $safe_commands = config_key($config, 'safe_commands');
5594
5595 unless(ref($config->{SUBSECTION}) eq "ARRAY") {
5596 ERROR "No volumes defined in configuration file";
5597 exit 2;
5598 }
5599
5600 # input validation (part 2, after config is initialized)
5601 @subvol_args = map { vinfo($_, $config) } @subvol_args;
5602 if($subvol_args_init) {
5603 foreach(@subvol_args) {
5604 unless(vinfo_init_root($_)) {
5605 ERROR "Failed to fetch subvolume detail for '$_->{PRINT}'" , @stderr;
5606 exit 1;
5607 }
5608 if(defined($_->{NODE_SUBDIR})) {
5609 ERROR "Argument is not a subvolume: $_->{PATH}";
5610 exit 1;
5611 }
5612 if(($subvol_args_init =~ /deny_root_subvol/) && $_->{node}{is_root}) {
5613 ERROR "Subvolume is btrfs root: $_->{PATH}";
5614 exit 1;
5615 }
5616 if(($subvol_args_init =~ /restrict_same_fs/) && (not _is_same_fs_tree($subvol_args[0]->{node}, $_->{node}))) {
5617 ERROR "Subvolumes are not on the same btrfs filesystem!";
5618 exit 1;
5619 }
5620 }
5621 }
5622
5623
5624 if($action_diff)
5625 {
5626 #
5627 # print snapshot diff (btrfs find-new)
5628 #
5629 my $src_vol = $subvol_args[0];
5630 my $target_vol = $subvol_args[1];
5631
5632 # NOTE: in some cases "cgen" differs from "gen", even for read-only snapshots (observed: gen=cgen+1)
5633 my $lastgen = $src_vol->{node}{gen} + 1;
5634
5635 # dump files, sorted and unique
5636 my $ret = btrfs_subvolume_find_new($target_vol, $lastgen);
5637 exit 1 unless(ref($ret));
5638
5639 INFO "Listing changed files for subvolume: $target_vol->{PRINT} (gen=$target_vol->{node}{gen})";
5640 INFO "Starting at generation after subvolume: $src_vol->{PRINT} (gen=$src_vol->{node}{gen})";
5641 INFO "Listing files modified within generation range: [$lastgen..$target_vol->{node}{gen}]";
5642 DEBUG "Newest file generation (transid marker) was: $ret->{transid_marker}";
5643
5644 my $files = $ret->{files};
5645 my $total_len = 0;
5646 my @data;
5647 foreach my $name (sort keys %$files) {
5648 my $finfo = $files->{$name};
5649 $total_len += $finfo->{len};
5650 push @data, {
5651 flags => ($finfo->{new} ? '+' : '.') .
5652 ($finfo->{flags}->{COMPRESS} ? 'c' : '.') .
5653 ($finfo->{flags}->{INLINE} ? 'i' : '.'),
5654 count => scalar(keys(%{$finfo->{gen}})),
5655 size => print_size($finfo->{len}),
5656 file => $name,
5657 };
5658 }
5659
5660 my $raw = ($output_format && $output_format eq "raw");
5661 print_formatted("diff", \@data, paragraph => 1);
5662 print "Total size: " . print_size($total_len) . "\n" unless($raw);
5663 exit 0;
5664 }
5665
5666
5667 if($action_extents)
5668 {
5669 #
5670 # print extents diff (filefrag)
5671 #
5672
5673 # check system requirements
5674 my $extentmap_fn;
5675 if($dryrun) {
5676 $extentmap_fn = sub {
5677 INFO("Fetching extent information (dryrun) for: $_[0]->{PRINT}");
5678 return undef;
5679 };
5680 }
5681 elsif(eval_quiet { require IO::AIO; }) {
5682 # this is slightly faster (multithreaded) than filefrag
5683 $extentmap_fn=\&aio_extentmap;
5684 }
5685 elsif(check_exe("filefrag")) {
5686 INFO "IO::AIO module not present, falling back to 'filefrag' (slower)";
5687 $extentmap_fn=\&filefrag_extentmap;
5688 }
5689 else {
5690 ERROR 'Please install either "IO::AIO" perl module or "filefrag" (from e2fsprogs package)';
5691 exit 1;
5692 }
5693 INFO "Extent map caching disabled (consider setting \"cache_dir\" configuration option)" unless(config_key($config, 'cache_dir'));
5694
5695 # resolve related subvolumes
5696 my @resolved_vol;
5697 if($extents_related) {
5698 # add all related subvolumes
5699 foreach my $svol (@subvol_args) {
5700 my $svol_gen = $svol->{node}{readonly} ? $svol->{node}{cgen} : $svol->{node}{gen};
5701 my @related = map({ vinfo_resolved_all_mountpoints($_, $svol->{VINFO_MOUNTPOINT}) // () }
5702 _related_nodes($svol->{node})); # includes $svol
5703 push @resolved_vol, @related;
5704 }
5705 }
5706 else {
5707 @resolved_vol = @subvol_args;
5708 }
5709
5710 my @data;
5711 # print results on ctrl-c
5712 $SIG{INT} = sub {
5713 print STDERR "\nERROR: Caught SIGINT, dumping incomplete list:\n";
5714 print_formatted("extent_diff", \@data);
5715 exit 1;
5716 };
5717
5718 my $do_diff = ($action_extents eq "diff");
5719 my $prev_data;
5720 # sort by gen for r/w subvolumes, cgen on readonly subvolumes, as
5721 # "gen" is increased on readonly subvolume when snapshotted.
5722 # crawl descending, but display ascending (unshift):
5723 foreach my $vol (sort { ($b->{node}{readonly} ? $b->{node}{cgen} : $b->{node}{gen}) <=>
5724 ($a->{node}{readonly} ? $a->{node}{cgen} : $a->{node}{gen}) }
5725 @resolved_vol) {
5726 if($prev_data && ($prev_data->{_vinfo}{node}{id} == $vol->{node}{id})) {
5727 INFO "Skipping duplicate of \"$prev_data->{_vinfo}{PRINT}\": $vol->{PRINT}";
5728 next;
5729 }
5730
5731 # read extent map
5732 if($vol->{EXTENTMAP} = read_extentmap_cache($vol)) {
5733 INFO "Using cached extent map: $vol->{PRINT}";
5734 } else {
5735 $vol->{EXTENTMAP} = $extentmap_fn->($vol);
5736 write_extentmap_cache($vol);
5737 }
5738 next unless($vol->{EXTENTMAP});
5739
5740 if($do_diff && $prev_data) {
5741 my $diff_map = extentmap_diff($prev_data->{_vinfo}{EXTENTMAP}, $vol->{EXTENTMAP});
5742 $prev_data->{diff} = print_size(extentmap_size($diff_map));
5743 }
5744 $prev_data = {
5745 %{$vol->{node}}, # copy node
5746 total => print_size(extentmap_size($vol->{EXTENTMAP})),
5747 subvol => $vol->{PRINT},
5748 _vinfo => $vol,
5749 };
5750 unshift @data, $prev_data;
5751 }
5752
5753 my @universe_set = map $_->{_vinfo}{EXTENTMAP}, @data;
5754 unless(scalar(@universe_set)) {
5755 ERROR "No extent map data, exiting";
5756 exit -1;
5757 }
5758
5759 my @summary;
5760 INFO "Calculating union of " . scalar(@data) . " subvolumes";
5761 push @summary, {
5762 a => "Union (" . scalar(@data) . " subvolumes):",
5763 b => print_size(extentmap_size(extentmap_merge(@universe_set)))
5764 };
5765
5766 INFO "Calculating set-exclusive size for " . scalar(@data) . " subvolumes";
5767 foreach my $d (@data) {
5768 my $vol = $d->{_vinfo};
5769 DEBUG "Calculating exclusive for: $vol->{PRINT}";
5770 my @others = grep { $_ != $vol->{EXTENTMAP} } @universe_set;
5771 $d->{exclusive} = print_size(extentmap_size(extentmap_diff($vol->{EXTENTMAP}, extentmap_merge(@others)))),
5772 }
5773
5774 if(scalar(@filter_vf)) {
5775 INFO "Calculating set difference (X \\ A)";
5776 my @excl;
5777 my @others;
5778 foreach(@data) {
5779 if(vinfo_match(\@filter_vf, $_->{_vinfo})) {
5780 $_->{set} = "X";
5781 push @excl, $_->{_vinfo}{EXTENTMAP};
5782 } else {
5783 $_->{set} = "A";
5784 push @others, $_->{_vinfo}{EXTENTMAP};
5785 }
5786 }
5787 push @summary, {
5788 a => "Exclusive data ( X \\ A ):",
5789 b => print_size(extentmap_size(extentmap_diff(extentmap_merge(@excl), extentmap_merge(@others)))),
5790 };
5791 }
5792 unless($do_diff) {
5793 @data = sort { $a->{subvol} cmp $b->{subvol} } @data;
5794 }
5795
5796 INFO "Printing extents map set difference: (extents \\ extents-on-prev-line)" if $do_diff;
5797 print_formatted("extent_diff", \@data, paragraph => 1);
5798 print_formatted({ table => [ qw( a b ) ], RALIGN => { b=>1 } },
5799 \@summary, output_format => "table", no_header => 1);
5800 exit 0;
5801 }
5802
5803
5804 if($action_ls)
5805 {
5806 #
5807 # print accessible subvolumes for local path
5808 #
5809 my $exit_status = 0;
5810 my %data_uniq;
5811 foreach my $root_vol (@subvol_args) {
5812 my ($root_path, $mountpoint) = vinfo_mountpoint($root_vol);
5813 unless($mountpoint) {
5814 ERROR "Failed to read filesystem details for: $root_vol->{PRINT}", @stderr;
5815 $exit_status = 1;
5816 next;
5817 }
5818 $root_vol = vinfo($root_vol->{URL_PREFIX} . $root_path, $config);
5819 INFO "Listing subvolumes for directory: $root_vol->{PRINT}";
5820
5821 my @search = ( $mountpoint );
5822 while(my $mnt = shift @search) {
5823 unshift @search, @{$mnt->{SUBTREE}} if($mnt->{SUBTREE});
5824 next if($mnt->{fs_type} ne "btrfs");
5825
5826 my $vol = vinfo($root_vol->{URL_PREFIX} . $mnt->{mount_point}, $config);
5827 unless(vinfo_init_root($vol)) {
5828 ERROR "Failed to fetch subvolume detail for: $vol->{PRINT}", @stderr;
5829 $exit_status = 1;
5830 next;
5831 }
5832
5833 my $subvol_list = vinfo_subvol_list($vol);
5834 my $count_added = 0;
5835 foreach my $svol ($vol, @$subvol_list) {
5836 my $svol_path = $svol->{PATH};
5837 $svol_path =~ s/^\/\//\//; # sanitize "//" (see vinfo_child)
5838
5839 next unless($root_path eq "/" || $svol_path =~ /^\Q$root_path\E(\/|\z)/);
5840 if(_find_mountpoint($mnt, $svol_path) ne $mnt) {
5841 DEBUG "Subvolume is hidden by another mount point: $svol->{PRINT}";
5842 next;
5843 }
5844
5845 $data_uniq{$svol->{PRINT}} = {
5846 %{$svol->{node}}, # copy node
5847 top => $svol->{node}{top_level}, # alias (narrow column)
5848 mount_point => $svol->{VINFO_MOUNTPOINT}{PATH},
5849 mount_source => $svol->{node}{TREE_ROOT}{mount_source},
5850 mount_subvolid => $mnt->{MNTOPS}{subvolid},
5851 mount_subvol => $mnt->{MNTOPS}{subvol},
5852 subvolume_path => $svol->{node}{path},
5853 subvolume_rel_path => $svol->{node}{REL_PATH},
5854 url => $svol->{URL},
5855 host => $svol->{HOST},
5856 path => $svol_path,
5857 flags => ($svol->{node}{readonly} ? "readonly" : undef),
5858 };
5859 $count_added++;
5860 }
5861 DEBUG "Listing $count_added/" . (scalar(@$subvol_list) + 1) . " subvolumes for btrfs mount: $vol->{PRINT}";
5862 }
5863 }
5864
5865 my @sorted = sort { (($a->{host} // "") cmp ($b->{host} // "")) ||
5866 ($a->{mount_point} cmp $b->{mount_point}) ||
5867 ($a->{path} cmp $b->{path})
5868 } values %data_uniq;
5869 $output_format ||= "short";
5870 print_formatted("fs_list", \@sorted, no_header => !scalar(@sorted));
5871
5872 exit $exit_status;
5873 }
5874
5875
5876 #
5877 # try exclusive lock if set in config or command-line option
5878 #
5879 $lockfile //= config_key($config, "lockfile");
5880 if(defined($lockfile) && (not $dryrun)) {
5881 unless(open(LOCKFILE, '>>', $lockfile)) {
5882 # NOTE: the lockfile is never deleted by design
5883 ERROR "Failed to open lock file '$lockfile': $!";
5884 exit 3;
5885 }
5886 unless(flock(LOCKFILE, 6)) { # exclusive, non-blocking (LOCK_EX | LOCK_NB)
5887 ERROR "Failed to take lock (another btrbk instance is running): $lockfile";
5888 exit 3;
5889 }
5890 }
5891
5892
5893 if($action_archive)
5894 {
5895 #
5896 # archive (clone) tree
5897 #
5898 # NOTE: This is intended to work without a config file! The only
5899 # thing used from the configuration is the SSH and transaction log
5900 # stuff.
5901 #
5902 init_transaction_log(config_key($config, "transaction_log"),
5903 config_key($config, "transaction_syslog"));
5904
5905 my $src_root = $subvol_args[0] || die;
5906 my $archive_root = $subvol_args[1] || die;
5907
5908 # FIXME: add command line options for preserve logic
5909 $config->{SUBSECTION} = []; # clear configured subsections, we build them dynamically
5910
5911 unless(vinfo_init_root($src_root)) {
5912 ERROR "Failed to fetch subvolume detail for '$src_root->{PRINT}'", @stderr;
5913 exit 1;
5914 }
5915 unless($archive_raw ? vinfo_init_raw_root($archive_root) : vinfo_init_root($archive_root)) {
5916 ERROR "Failed to fetch " . ($archive_raw ? "raw target metadata" : "subvolume detail") . " for '$archive_root->{PRINT}'", @stderr;
5917 exit 1;
5918 }
5919
5920 my %name_uniq;
5921 my @subvol_list = @{vinfo_subvol_list($src_root)};
5922 my @sorted = sort { ($a->{subtree_depth} <=> $b->{subtree_depth}) || ($a->{SUBVOL_DIR} cmp $b->{SUBVOL_DIR}) } @subvol_list;
5923 foreach my $vol (@sorted) {
5924 next unless($vol->{node}{readonly});
5925 my $snapshot_name = $vol->{node}{BTRBK_BASENAME};
5926 unless(defined($snapshot_name)) {
5927 WARN "Skipping subvolume (not a btrbk subvolume): $vol->{PRINT}";
5928 next;
5929 }
5930 my $subvol_dir = $vol->{SUBVOL_DIR};
5931 next if($name_uniq{"$subvol_dir/$snapshot_name"});
5932 $name_uniq{"$subvol_dir/$snapshot_name"} = 1;
5933 my $droot_url = $archive_root->{URL} . ($subvol_dir eq "" ? "" : "/$subvol_dir");
5934 my $sroot_url = $src_root->{URL} . ($subvol_dir eq "" ? "" : "/$subvol_dir");
5935 my $config_sroot = { CONTEXT => "archive_source",
5936 PARENT => $config,
5937 url => $sroot_url, # ABORTED() needs this
5938 snapshot_name => $snapshot_name,
5939 };
5940 my $config_droot = { CONTEXT => "archive_target",
5941 PARENT => $config_sroot,
5942 target_type => ($archive_raw ? "raw" : "send-receive"), # macro_send_receive checks this
5943 url => $droot_url, # ABORTED() needs this
5944 };
5945 $config_sroot->{SUBSECTION} = [ $config_droot ];
5946 push(@{$config->{SUBSECTION}}, $config_sroot);
5947
5948 my $sroot = vinfo($sroot_url, $config_sroot);
5949 vinfo_assign_config($sroot);
5950 unless(vinfo_init_root($sroot)) {
5951 ABORTED($sroot, "Failed to fetch subvolume detail");
5952 WARN "Skipping archive source \"$sroot->{PRINT}\": " . ABORTED_TEXT($sroot), @stderr;
5953 next;
5954 }
5955
5956 my $droot = vinfo($droot_url, $config_droot);
5957 vinfo_assign_config($droot);
5958 unless($archive_raw ? vinfo_init_raw_root($droot) : vinfo_init_root($droot)) {
5959 DEBUG "Failed to fetch " . ($archive_raw ? "raw target metadata" : "subvolume detail") . " for '$droot->{PRINT}'";
5960 unless(system_mkdir($droot)) {
5961 ABORTED($droot, "Failed to create directory: $droot->{PRINT}/");
5962 WARN "Skipping archive target \"$droot->{PRINT}\": " . ABORTED_TEXT($droot), @stderr;
5963 next;
5964 }
5965 $droot->{SUBDIR_CREATED} = 1;
5966 if($dryrun) {
5967 # we need to fake this directory on dryrun
5968 $droot->{node} = $archive_root->{node};
5969 $droot->{NODE_SUBDIR} = $subvol_dir;
5970 $droot->{VINFO_MOUNTPOINT} = $archive_root->{VINFO_MOUNTPOINT};
5971 $realpath_cache{$droot->{URL}} = $droot->{PATH};
5972 }
5973 else {
5974 # after directory is created, try to init again
5975 unless($archive_raw ? vinfo_init_raw_root($droot) : vinfo_init_root($droot)) {
5976 ABORTED($droot, "Failed to fetch subvolume detail");
5977 WARN "Skipping archive target \"$droot->{PRINT}\": " . ABORTED_TEXT($droot), @stderr;
5978 next;
5979 }
5980 }
5981 }
5982 if(_is_same_fs_tree($droot->{node}, $vol->{node})) {
5983 ERROR "Source and target subvolumes are on the same btrfs filesystem!";
5984 exit 1;
5985 }
5986 }
5987
5988 # translate archive_exclude globs, add to exclude args
5989 my $archive_exclude = config_key($config, 'archive_exclude') // [];
5990 push @exclude_vf, map(vinfo_filter_statement($_), (@$archive_exclude));
5991
5992 # create archives
5993 my $schedule_results = [];
5994 my $aborted;
5995 foreach my $sroot (vinfo_subsection($config, 'archive_source')) {
5996 if($aborted) {
5997 # abort all subsequent sources on any abort (we don't want to go on hammering on "disk full" errors)
5998 ABORTED($sroot, $aborted);
5999 next;
6000 }
6001 my $snapshot_name = config_key($sroot, "snapshot_name") // die;
6002
6003 # skip on archive_exclude and --exclude option
6004 if(vinfo_match(\@exclude_vf, $sroot) ||
6005 vinfo_match(\@exclude_vf, vinfo_child($sroot, $snapshot_name)))
6006 {
6007 ABORTED($sroot, "skip_archive_exclude", "Match on exclude pattern");
6008 INFO "Skipping archive subvolumes \"$sroot->{PRINT}/${snapshot_name}.*\": " . ABORTED_TEXT($sroot);
6009 next;
6010 }
6011
6012 foreach my $droot (vinfo_subsection($sroot, 'archive_target')) {
6013 INFO "Archiving subvolumes: $sroot->{PRINT}/${snapshot_name}.*";
6014 macro_archive_target($sroot, $droot, $snapshot_name, { results => $schedule_results });
6015 if(IS_ABORTED($droot)) {
6016 # also abort $sroot
6017 $aborted = "At least one target aborted earlier";
6018 ABORTED($sroot, $aborted);
6019 WARN "Skipping archiving of \"$sroot->{PRINT}/\": " . ABORTED_TEXT($sroot);
6020 last;
6021 }
6022 }
6023 }
6024
6025 # delete archives
6026 my $del_schedule_results;
6027 if($preserve_backups) {
6028 INFO "Preserving all archives (option \"-p\" or \"-r\" present)";
6029 }
6030 else
6031 {
6032 $del_schedule_results = [];
6033 foreach my $sroot (vinfo_subsection($config, 'archive_source')) {
6034 my $snapshot_name = config_key($sroot, "snapshot_name") // die;
6035 foreach my $droot (vinfo_subsection($sroot, 'archive_target')) {
6036 INFO "Cleaning archive: $droot->{PRINT}/${snapshot_name}.*";
6037 macro_delete($droot, $snapshot_name, $droot,
6038 { preserve => config_preserve_hash($droot, "archive"),
6039 results => $del_schedule_results,
6040 result_hints => { topic => "archive", root_path => $droot->{PATH} },
6041 },
6042 commit => config_key($droot, "btrfs_commit_delete"),
6043 type => "delete_archive",
6044 qgroup => { destroy => config_key($droot, "archive_qgroup_destroy"),
6045 type => "qgroup_destroy_archive" },
6046 );
6047 }
6048 }
6049 }
6050
6051
6052 my $exit_status = exit_status($config);
6053 my $time_elapsed = time - $start_time;
6054 INFO "Completed within: ${time_elapsed}s (" . localtime(time) . ")";
6055 action("finished",
6056 status => $exit_status ? "partial" : "success",
6057 duration => $time_elapsed,
6058 message => $exit_status ? "At least one backup task aborted" : undef,
6059 );
6060 close_transaction_log();
6061
6062 unless($quiet)
6063 {
6064 # print scheduling results
6065 if($print_schedule) {
6066 my @data = map { { %$_, vinfo_prefixed_keys("", $_->{value}) }; } @$schedule_results;
6067 print_formatted("schedule", \@data, title => "ARCHIVE SCHEDULE", paragraph => 1);
6068 }
6069
6070 if($print_schedule && $del_schedule_results) {
6071 my @data = map { { %$_, vinfo_prefixed_keys("", $_->{value}) }; } @$del_schedule_results;
6072 print_formatted("schedule", \@data, title => "DELETE SCHEDULE", paragraph => 1);
6073 }
6074
6075 # print summary
6076 $output_format ||= "custom";
6077 if($output_format eq "custom")
6078 {
6079 my @out;
6080 foreach my $sroot (vinfo_subsection($config, 'archive_source', 1)) {
6081 foreach my $droot (vinfo_subsection($sroot, 'archive_target', 1)) {
6082 my @subvol_out;
6083 if($droot->{SUBDIR_CREATED}) {
6084 push @subvol_out, "++. $droot->{PRINT}/";
6085 }
6086 foreach(@{$droot->{SUBVOL_RECEIVED} // []}) {
6087 my $create_mode = "***";
6088 $create_mode = ">>>" if($_->{parent});
6089 $create_mode = "!!!" if($_->{ERROR});
6090 push @subvol_out, "$create_mode $_->{received_subvolume}->{PRINT}";
6091 }
6092 foreach(@{$droot->{SUBVOL_DELETED} // []}) {
6093 push @subvol_out, "--- $_->{PRINT}";
6094 }
6095 if(IS_ABORTED($droot, "abort_") || IS_ABORTED($sroot, "abort_")) {
6096 push @subvol_out, "!!! Target \"$droot->{PRINT}\" aborted: " . (ABORTED_TEXT($droot) || ABORTED_TEXT($sroot));
6097 }
6098 elsif(IS_ABORTED($sroot, "skip_archive_exclude")) {
6099 push @subvol_out, "<archive_exclude>";
6100 }
6101 unless(@subvol_out) {
6102 push @subvol_out, "[-] $droot->{PRINT}/$sroot->{CONFIG}->{snapshot_name}.*";
6103 }
6104 push @out, "$sroot->{PRINT}/$sroot->{CONFIG}->{snapshot_name}.*", @subvol_out, "";
6105 }
6106 }
6107
6108 my @cmdline_options = map { "exclude: $_" } @exclude_cmdline;
6109 push @cmdline_options, "preserve: Preserved all archives" if($preserve_backups);
6110
6111 print_header(title => "Archive Summary",
6112 time => $start_time,
6113 options => \@cmdline_options,
6114 legend => [
6115 "++. created directory",
6116 "--- deleted subvolume",
6117 "*** received subvolume (non-incremental)",
6118 ">>> received subvolume (incremental)",
6119 "[-] no action",
6120 ],
6121 );
6122 print join("\n", @out);
6123 print_footer($config, $exit_status);
6124 }
6125 else
6126 {
6127 # print action log (without transaction start messages)
6128 my @data = grep { $_->{status} !~ /starting$/ } @transaction_log;
6129 print_formatted("transaction", \@data, title => "TRANSACTION LOG");
6130 }
6131 }
6132
6133 exit $exit_status;
6134 }
6135
6136
6137 #
6138 # expand subvolume globs (wildcards)
6139 #
6140 foreach my $config_vol (config_subsection($config, "volume")) {
6141 # read-in subvolume list (and expand globs) only if needed
6142 next unless(grep defined($_->{GLOB_CONTEXT}), @{$config_vol->{SUBSECTION}});
6143 my @vol_subsection_expanded;
6144 foreach my $config_subvol (config_subsection($config_vol, "subvolume")) {
6145 if($config_subvol->{GLOB_CONTEXT}) {
6146 my ($url_prefix, $globs) = check_url($config_subvol->{url}, accept_wildcards => 1);
6147 $globs =~ s/([^\*]*)\///;
6148 my $sroot_glob = vinfo($url_prefix . $1, $config_subvol);
6149
6150 INFO "Expanding wildcards: $sroot_glob->{PRINT}/$globs";
6151 unless(vinfo_init_root($sroot_glob)) {
6152 WARN "Failed to fetch subvolume detail for: $sroot_glob->{PRINT}", @stderr;
6153 WARN "No subvolumes found matching: $sroot_glob->{PRINT}/$globs";
6154 next;
6155 }
6156
6157 # support "*some*file*", "*/*"
6158 my $match = join('[^\/]*', map(quotemeta($_), split(/\*+/, $globs, -1)));
6159 TRACE "translated globs \"$globs\" to regex \"$match\"" if($do_trace);
6160 my $expand_count = 0;
6161 foreach my $vol (@{vinfo_subvol_list($sroot_glob, sort => 'path')})
6162 {
6163 if($vol->{node}{readonly}) {
6164 TRACE "skipping readonly subvolume: $vol->{PRINT}" if($do_trace);
6165 next;
6166 }
6167 unless($vol->{SUBVOL_PATH} =~ /^$match$/) {
6168 TRACE "skipping non-matching subvolume: $vol->{PRINT}" if($do_trace);
6169 next;
6170 }
6171 unless(defined(check_file($vol->{SUBVOL_PATH}, { relative => 1 }))) {
6172 WARN "Ambiguous subvolume path \"$vol->{SUBVOL_PATH}\" while expanding \"$globs\", ignoring";
6173 next;
6174 }
6175 INFO "Found source subvolume: $vol->{PRINT}";
6176 my %conf = ( %$config_subvol,
6177 url_glob => $config_subvol->{url},
6178 url => $vol->{URL},
6179 snapshot_name => $vol->{NAME}, # snapshot_name defaults to subvolume name
6180 );
6181 # deep copy of target subsection
6182 my @subsection_copy = map { { %$_, PARENT => \%conf }; } @{$config_subvol->{SUBSECTION}};
6183 $conf{SUBSECTION} = \@subsection_copy;
6184 push @vol_subsection_expanded, \%conf;
6185 $expand_count += 1;
6186 }
6187 unless($expand_count) {
6188 WARN "No subvolumes found matching: $sroot_glob->{PRINT}/$globs";
6189 }
6190 }
6191 else {
6192 push @vol_subsection_expanded, $config_subvol;
6193 }
6194 }
6195 $config_vol->{SUBSECTION} = \@vol_subsection_expanded;
6196 }
6197 TRACE(Data::Dumper->Dump([$config], ["config"])) if($do_trace && $do_dumper);
6198
6199
6200 #
6201 # create vinfo nodes (no readin yet)
6202 #
6203 foreach my $config_vol (config_subsection($config, "volume")) {
6204 my $sroot = $config_vol->{DUMMY} ? { CONFIG => $config_vol, PRINT => "*default*" } : vinfo($config_vol->{url}, $config_vol);
6205 vinfo_assign_config($sroot);
6206 foreach my $config_subvol (config_subsection($config_vol, "subvolume")) {
6207 my $svol = vinfo($config_subvol->{url}, $config_subvol);
6208 my $snapshot_dir = config_key($svol, "snapshot_dir");
6209 my $url;
6210 if(!defined($snapshot_dir)) {
6211 if($config_vol->{DUMMY}) {
6212 ABORTED($svol, "No snapshot_dir defined for subvolume");
6213 WARN "Skipping subvolume \"$svol->{PRINT}\": " . ABORTED_TEXT($svol);
6214 } else {
6215 $url = $sroot->{URL};
6216 }
6217 } elsif($snapshot_dir =~ /^\//) {
6218 $url = $svol->{URL_PREFIX} . $snapshot_dir;
6219 } else {
6220 if($config_vol->{DUMMY}) {
6221 ABORTED($svol, "Relative snapshot_dir path defined, but no volume context present");
6222 WARN "Skipping subvolume \"$svol->{PRINT}\": " . ABORTED_TEXT($svol);
6223 } else {
6224 $url = $sroot->{URL} . '/' . $snapshot_dir;
6225 }
6226 }
6227 $url //= "/dev/null"; # snaproot cannot be undef, even if ABORTED
6228 my $snaproot = vinfo($url, $config_subvol);
6229 vinfo_assign_config($svol, $snaproot);
6230 foreach my $config_target (@{$config_subvol->{SUBSECTION}}) {
6231 die unless($config_target->{CONTEXT} eq "target");
6232 my $droot = vinfo($config_target->{url}, $config_target);
6233 vinfo_assign_config($droot);
6234 }
6235 }
6236 }
6237
6238
6239 #
6240 # filter subvolumes matching command line arguments, handle noauto option
6241 #
6242 if(scalar @filter_vf)
6243 {
6244 foreach my $sroot (vinfo_subsection($config, 'volume', 1)) {
6245 my $found_vol = 0;
6246 if(vinfo_match(\@filter_vf, $sroot, flag_matched => '_matched')) {
6247 next;
6248 }
6249 foreach my $svol (vinfo_subsection($sroot, 'subvolume', 1)) {
6250 my $found_subvol = 0;
6251 my $snaproot = vinfo_snapshot_root($svol);
6252 my $snapshot_name = config_key($svol, "snapshot_name") // die;
6253 if(vinfo_match(\@filter_vf, $svol, flag_matched => '_matched') ||
6254 vinfo_match(\@filter_vf, vinfo_child($snaproot, $snapshot_name), flag_matched => '_matched'))
6255 {
6256 $found_vol = 1;
6257 next;
6258 }
6259 foreach my $droot (vinfo_subsection($svol, 'target', 1)) {
6260 if(vinfo_match(\@filter_vf, $droot, flag_matched => '_matched') ||
6261 vinfo_match(\@filter_vf, vinfo_child($droot, $snapshot_name), flag_matched => '_matched'))
6262 {
6263 $found_subvol = 1;
6264 $found_vol = 1;
6265 }
6266 else {
6267 ABORTED($droot, "skip_cmdline_filter", "No match on filter command line argument");
6268 DEBUG "Skipping target \"$droot->{PRINT}\": " . ABORTED_TEXT($droot);
6269 }
6270 }
6271 unless($found_subvol) {
6272 ABORTED($svol, "skip_cmdline_filter", "No match on filter command line argument");
6273 DEBUG "Skipping subvolume \"$svol->{PRINT}\": " . ABORTED_TEXT($svol);
6274 }
6275 }
6276 unless($found_vol) {
6277 ABORTED($sroot, "skip_cmdline_filter", "No match on filter command line argument");
6278 DEBUG "Skipping volume \"$sroot->{PRINT}\": " . ABORTED_TEXT($sroot);
6279 }
6280 }
6281 # make sure all args have a match
6282 my @nomatch = map { $_->{_matched} ? () : $_->{unparsed} } @filter_vf;
6283 if(@nomatch) {
6284 foreach(@nomatch) {
6285 ERROR "Filter argument \"$_\" does not match any volume, subvolume, target or group declaration";
6286 }
6287 exit 2;
6288 }
6289 $config->{CMDLINE_FILTER_LIST} = [ map { $_->{unparsed} } @filter_vf ];
6290 }
6291 elsif(not $action_config_print)
6292 {
6293 # no filter_args present, abort "noauto" contexts
6294 if(config_key($config, "noauto")) {
6295 WARN "Option \"noauto\" is set in global context, and no filter argument present, exiting";
6296 exit 0;
6297 }
6298 foreach my $sroot (vinfo_subsection($config, 'volume')) {
6299 if(config_key($sroot, "noauto")) {
6300 ABORTED($sroot, "skip_noauto", 'option "noauto" is set');
6301 DEBUG "Skipping volume \"$sroot->{PRINT}\": " . ABORTED_TEXT($sroot);
6302 next;
6303 }
6304 foreach my $svol (vinfo_subsection($sroot, 'subvolume')) {
6305 if(config_key($svol, "noauto")) {
6306 ABORTED($svol, "skip_noauto", 'option "noauto" is set');
6307 DEBUG "Skipping subvolume \"$svol->{PRINT}\": " . ABORTED_TEXT($svol);
6308 next;
6309 }
6310 foreach my $droot (vinfo_subsection($svol, 'target')) {
6311 if(config_key($droot, "noauto")) {
6312 ABORTED($droot, "skip_noauto", 'option "noauto" is set');
6313 DEBUG "Skipping target \"$droot->{PRINT}\": " . ABORTED_TEXT($droot);
6314 }
6315 }
6316 }
6317 }
6318 }
6319
6320 if(scalar @exclude_vf)
6321 {
6322 # handle --exclude command line option
6323 foreach my $sroot (vinfo_subsection($config, 'volume')) {
6324 if(my $ff = vinfo_match(\@exclude_vf, $sroot)) {
6325 ABORTED($sroot, "skip_cmdline_exclude", "command line argument \"--exclude=$ff->{unparsed}\"");
6326 DEBUG "Skipping volume \"$sroot->{PRINT}\": " . ABORTED_TEXT($sroot);
6327 next;
6328 }
6329 my $all_svol_aborted = 1;
6330 foreach my $svol (vinfo_subsection($sroot, 'subvolume')) {
6331 my $snaproot = vinfo_snapshot_root($svol);
6332 my $snapshot_name = config_key($svol, "snapshot_name") // die;
6333 if(my $ff = (vinfo_match(\@exclude_vf, $svol) ||
6334 vinfo_match(\@exclude_vf, vinfo_child($snaproot, $snapshot_name))))
6335 {
6336 ABORTED($svol, "skip_cmdline_exclude", "command line argument \"--exclude=$ff->{unparsed}\"");
6337 DEBUG "Skipping subvolume \"$svol->{PRINT}\": " . ABORTED_TEXT($svol);
6338 next;
6339 }
6340 $all_svol_aborted = 0;
6341 foreach my $droot (vinfo_subsection($svol, 'target')) {
6342 if(my $ff = (vinfo_match(\@exclude_vf, $droot) ||
6343 vinfo_match(\@exclude_vf, vinfo_child($droot, $snapshot_name))))
6344 {
6345 ABORTED($droot, "skip_cmdline_exclude", "command line argument \"--exclude=$ff->{unparsed}\"");
6346 DEBUG "Skipping target \"$droot->{PRINT}\": " . ABORTED_TEXT($droot);
6347 next;
6348 }
6349 }
6350 }
6351 if($all_svol_aborted) {
6352 ABORTED($sroot, "skip_cmdline_exclude", "All subvolumes excluded");
6353 DEBUG "Skipping volume \"$sroot->{PRINT}\": " . ABORTED_TEXT($sroot);
6354 }
6355 }
6356 }
6357
6358 if($action_usage)
6359 {
6360 #
6361 # print filesystem information
6362 #
6363 my @data;
6364 my %usage_cache;
6365 my %processed;
6366 my $push_data = sub {
6367 my ($vol, $type) = @_;
6368 return if $processed{$vol->{URL}};
6369 my $mountpoint = vinfo_mountpoint($vol, fs_type => 'btrfs');
6370 return unless($mountpoint);
6371 my $mount_source = $mountpoint->{mount_source};
6372 my $mid = $vol->{MACHINE_ID} . $mount_source;
6373 $usage_cache{$mid} //= btrfs_filesystem_usage(vinfo($vol->{URL_PREFIX} . $mountpoint->{mount_point}, $vol->{CONFIG})) // {};
6374 push @data, { %{$usage_cache{$mid}},
6375 type => $type,
6376 mount_source => $mount_source,
6377 vinfo_prefixed_keys("", $vol),
6378 };
6379 $processed{$vol->{URL}} = 1;
6380 };
6381 foreach my $sroot (vinfo_subsection($config, 'volume')) {
6382 foreach my $svol (vinfo_subsection($sroot, 'subvolume')) {
6383 $push_data->($svol, "source");
6384 foreach my $droot (vinfo_subsection($svol, 'target')) {
6385 $push_data->($droot, "target");
6386 }
6387 }
6388 }
6389 @data = sort { $a->{url} cmp $b->{url} } @data;
6390 print_formatted("usage", \@data);
6391 exit exit_status($config);
6392 }
6393
6394
6395 if($action_config_print)
6396 {
6397 #
6398 # print configuration lines, machine readable
6399 #
6400 my %opts = (all => ($action_config_print eq "print-all"));
6401 my @out;
6402 push @out, config_dump_keys($config, %opts);
6403 my $indent = "";
6404 foreach my $sroot (vinfo_subsection($config, 'volume', 1)) {
6405 unless($sroot->{CONFIG}{DUMMY}) {
6406 push @out, "";
6407 push @out, "volume $sroot->{URL}";
6408 $indent .= "\t";
6409 push @out, config_dump_keys($sroot, prefix => $indent, %opts);
6410 }
6411 foreach my $svol (vinfo_subsection($sroot, 'subvolume', 1)) {
6412 push @out, "";
6413 push @out, "${indent}# subvolume $svol->{CONFIG}->{url_glob}" if(defined($svol->{CONFIG}->{url_glob}));
6414 push @out, "${indent}subvolume $svol->{URL}";
6415 $indent .= "\t";
6416 push @out, config_dump_keys($svol, prefix => $indent, %opts);
6417 foreach my $droot (vinfo_subsection($svol, 'target', 1)) {
6418 push @out, "";
6419 push @out, "${indent}target $droot->{CONFIG}->{target_type} $droot->{URL}";
6420 push @out, config_dump_keys($droot, prefix => "\t$indent", %opts);
6421 }
6422 $indent =~ s/\t//;
6423 }
6424 $indent = "";
6425 }
6426
6427 print_header(title => "Configuration Dump",
6428 config => $config,
6429 time => $start_time,
6430 );
6431
6432 print join("\n", @out) . "\n";
6433 exit exit_status($config);
6434 }
6435
6436
6437 if($action_list)
6438 {
6439 my @vol_data;
6440 my @subvol_data;
6441 my @target_data;
6442 my @mixed_data;
6443 my %target_uniq;
6444
6445 #
6446 # print configuration lines, machine readable
6447 #
6448 foreach my $sroot (vinfo_subsection($config, 'volume')) {
6449 my $volh = { vinfo_prefixed_keys("volume", $sroot) };
6450 push @vol_data, $volh;
6451
6452 foreach my $svol (vinfo_subsection($sroot, 'subvolume')) {
6453 my $snaproot = vinfo_snapshot_root($svol);
6454 my $subvolh = { %$volh,
6455 vinfo_prefixed_keys("source", $svol),
6456 snapshot_path => $snaproot->{PATH},
6457 snapshot_name => config_key($svol, "snapshot_name"),
6458 snapshot_preserve => format_preserve_matrix(config_preserve_hash($svol, "snapshot")),
6459 };
6460 push @subvol_data, $subvolh;
6461
6462 my $found = 0;
6463 foreach my $droot (vinfo_subsection($svol, 'target')) {
6464 my $targeth = { %$subvolh,
6465 vinfo_prefixed_keys("target", $droot),
6466 target_preserve => format_preserve_matrix(config_preserve_hash($droot, "target")),
6467 target_type => $droot->{CONFIG}{target_type}, # "send-receive" or "raw"
6468 };
6469 if($action_list eq "target") {
6470 next if($target_uniq{$droot->{URL}});
6471 $target_uniq{$droot->{URL}} = 1;
6472 }
6473 push @target_data, $targeth;
6474 push @mixed_data, $targeth;
6475 $found = 1;
6476 }
6477 # make sure the subvol is always printed (even if no targets around)
6478 push @mixed_data, $subvolh unless($found);
6479 }
6480 }
6481 if($action_list eq "volume") {
6482 print_formatted("config_volume", \@vol_data);
6483 }
6484 elsif($action_list eq "source") {
6485 print_formatted("config_source", \@subvol_data);
6486 }
6487 elsif($action_list eq "target") {
6488 print_formatted("config_target", \@target_data);
6489 }
6490 elsif($action_list eq "config") {
6491 print_formatted("config", \@mixed_data);
6492 }
6493 else {
6494 die "unknown action_list=$action_list";
6495 }
6496 exit exit_status($config);
6497 }
6498
6499
6500 #
6501 # fill vinfo hash, basic checks on configuration
6502 #
6503
6504 # read volume btrfs tree, and make sure subvolume exist
6505 foreach my $sroot (vinfo_subsection($config, 'volume')) {
6506 DEBUG "Initializing volume section: $sroot->{PRINT}";
6507 unless(scalar(vinfo_subsection($sroot, 'subvolume', 1))) {
6508 WARN "No subvolume configured for \"volume $sroot->{URL}\"";
6509 }
6510 foreach my $svol (vinfo_subsection($sroot, 'subvolume')) {
6511 DEBUG "Initializing subvolume section: $svol->{PRINT}";
6512 unless(vinfo_init_root($svol)) {
6513 ABORTED($svol, "Failed to fetch subvolume detail");
6514 WARN "Skipping subvolume \"$svol->{PRINT}\": " . ABORTED_TEXT($svol), @stderr;
6515 next;
6516 }
6517 if((not $svol->{node}{uuid}) || ($svol->{node}{uuid} eq '-')) {
6518 ABORTED($svol, "subvolume has no UUID");
6519 ERROR "Skipping subvolume \"$svol->{PRINT}\": " . ABORTED_TEXT($svol);
6520 next;
6521 }
6522 if($svol->{node}{readonly}) {
6523 ABORTED($svol, "subvolume is readonly");
6524 WARN "Skipping subvolume \"$svol->{PRINT}\": " . ABORTED_TEXT($svol);
6525 next;
6526 }
6527 if($svol->{node}{received_uuid} ne '-') {
6528 ABORTED($svol, "\"Received UUID\" is set");
6529 WARN "Skipping subvolume \"$svol->{PRINT}\": " . ABORTED_TEXT($svol);
6530 next;
6531 }
6532
6533 my $snaproot = vinfo_snapshot_root($svol);
6534 unless(vinfo_init_root($snaproot)) {
6535 ABORTED($svol, "Failed to fetch subvolume detail for snapshot_dir");
6536 WARN "Skipping subvolume \"$svol->{PRINT}\": " . ABORTED_TEXT($svol), @stderr;
6537 next;
6538 }
6539 unless(_is_same_fs_tree($snaproot->{node}, $svol->{node})) {
6540 ABORTED($svol, "Snapshot path is not on same filesystem");
6541 WARN "Skipping subvolume \"$svol->{PRINT}\": " . ABORTED_TEXT($svol);
6542 next;
6543 }
6544 }
6545 }
6546
6547 # read target btrfs tree
6548 if($action_run && $skip_backups && $preserve_snapshots && $preserve_backups) {
6549 # if running "btrbk snapshot --preserve", there is no need to
6550 # initialize targets, and we don't want to fail on missing targets.
6551 DEBUG "Skipping target tree readin (preserving all snapshots and backups)";
6552 }
6553 else {
6554 foreach my $sroot (vinfo_subsection($config, 'volume')) {
6555 foreach my $svol (vinfo_subsection($sroot, 'subvolume')) {
6556 foreach my $droot (vinfo_subsection($svol, 'target')) {
6557 DEBUG "Initializing target section: $droot->{PRINT}";
6558 my $target_type = $droot->{CONFIG}->{target_type} || die;
6559 if($target_type eq "send-receive")
6560 {
6561 unless(vinfo_init_root($droot)) {
6562 ABORTED($droot, "Failed to fetch subvolume detail");
6563 WARN "Skipping target \"$droot->{PRINT}\": " . ABORTED_TEXT($droot), @stderr;
6564 next;
6565 }
6566 }
6567 elsif($target_type eq "raw")
6568 {
6569 unless(vinfo_init_raw_root($droot)) {
6570 ABORTED($droot, "Failed to fetch raw target metadata");
6571 WARN "Skipping target \"$droot->{PRINT}\": " . ABORTED_TEXT($droot), @stderr;
6572 next;
6573 }
6574 }
6575
6576 if($config_override{FAILSAFE_PRESERVE}) {
6577 ABORTED($droot, $config_override{FAILSAFE_PRESERVE});
6578 WARN "Skipping target \"$droot->{PRINT}\": " . ABORTED_TEXT($droot);
6579 }
6580 }
6581 }
6582 }
6583 }
6584
6585 # check for duplicate snapshot locations
6586 foreach my $sroot (vinfo_subsection($config, 'volume')) {
6587 foreach my $svol (vinfo_subsection($sroot, 'subvolume')) {
6588 my $snapshot_basename = config_key($svol, "snapshot_name") // die;
6589
6590 # check for duplicate snapshot locations
6591 if(config_key($svol, "snapshot_create")) {
6592 my $snaproot = vinfo_snapshot_root($svol);
6593 my $snaproot_subdir_path = (defined($snaproot->{NODE_SUBDIR}) ? $snaproot->{NODE_SUBDIR} . '/' : "") . $snapshot_basename;
6594 if(my $prev = $snaproot->{node}->{_SNAPSHOT_CHECK}->{$snaproot_subdir_path}) {
6595 ERROR "Subvolume \"$prev\" and \"$svol->{PRINT}\" will create same snapshot: $snaproot->{PRINT}/${snapshot_basename}.*";
6596 ERROR "Please fix \"snapshot_name\" configuration options!";
6597 exit 1;
6598 }
6599 $snaproot->{node}->{_SNAPSHOT_CHECK}->{$snaproot_subdir_path} = $svol->{PRINT};
6600 }
6601
6602 # check for duplicate target locations
6603 foreach my $droot (vinfo_subsection($svol, 'target')) {
6604 my $droot_subdir_path = (defined($droot->{NODE_SUBDIR}) ? $droot->{NODE_SUBDIR} . '/' : "") . $snapshot_basename;
6605 if(my $prev = $droot->{node}->{_BACKUP_CHECK}->{$droot_subdir_path}) {
6606 ERROR "Subvolume \"$prev\" and \"$svol->{PRINT}\" will create same backup target: $droot->{PRINT}/${snapshot_basename}.*";
6607 ERROR "Please fix \"snapshot_name\" or \"target\" configuration options!";
6608 exit 1;
6609 }
6610 $droot->{node}->{_BACKUP_CHECK}->{$droot_subdir_path} = $svol->{PRINT};
6611 }
6612 }
6613 }
6614
6615
6616 if($action_origin)
6617 {
6618 #
6619 # print origin information
6620 #
6621 my $vol = $subvol_args[0] || die;
6622 my $lines = [];
6623 _origin_tree("", $vol->{node}, $lines);
6624
6625 $output_format ||= "custom";
6626 if($output_format eq "custom") {
6627 print_header(title => "Origin Tree",
6628 config => $config,
6629 time => $start_time,
6630 legend => [
6631 "^-- : parent subvolume",
6632 "newline : received-from relationship with subvolume (identical content)",
6633 ]
6634 );
6635 print join("\n", map { $_->{tree} } @$lines) . "\n";
6636 }
6637 else {
6638 print_formatted('origin_tree', $lines );
6639 }
6640 exit 0;
6641 }
6642
6643
6644 if($action_resolve)
6645 {
6646 my @data;
6647 my %stats = ( snapshots => 0, backups => 0, correlated => 0, incomplete => 0, orphaned => 0 );
6648 foreach my $sroot (vinfo_subsection($config, 'volume')) {
6649 foreach my $svol (vinfo_subsection($sroot, 'subvolume')) {
6650 my $snaproot = vinfo_snapshot_root($svol);
6651 my $snapshot_name = config_key($svol, "snapshot_name") // die;
6652 my @snapshots = @{vinfo_subvol_list($snaproot, readonly => 1, btrbk_direct_leaf => $snapshot_name, sort => 'path')};
6653 my %svol_data = (
6654 vinfo_prefixed_keys("source", $svol),
6655 snapshot_name => $snapshot_name,
6656 );
6657 my @sdata = map +{
6658 %svol_data,
6659 type => "snapshot",
6660 status => ($_->{node}{cgen} == $svol->{node}{gen}) ? "up-to-date" : "",
6661 vinfo_prefixed_keys("snapshot", $_),
6662 _vinfo => $_,
6663 _btrbk_date => $_->{node}{BTRBK_DATE},
6664 }, @snapshots;
6665
6666 my %svol_stats_data = (
6667 %svol_data,
6668 snapshot_subvolume => "$snaproot->{PATH}/$snapshot_name.*",
6669 snapshot_status => (grep { $_->{status} eq "up-to-date" } @sdata) ? "up-to-date" : "",
6670 snapshots => scalar(@sdata),
6671 );
6672 $stats{snapshots} += scalar(@sdata);
6673
6674 my (@bdata, @ldata, @stdata);
6675 foreach my $droot (vinfo_subsection($svol, 'target')) {
6676 my %dstats = ( backups => 0, correlated => 0, orphaned => 0, incomplete => 0, uptodate => 0 );
6677 my $latest_backup;
6678 foreach my $target_vol (@{vinfo_subvol_list($droot, btrbk_direct_leaf => $snapshot_name, sort => 'path')}) {
6679 my $target_data = {
6680 %svol_data,
6681 type => "backup",
6682 target_type => $target_vol->{CONFIG}{target_type}, # "send-receive" or "raw"
6683 vinfo_prefixed_keys("target", $target_vol),
6684 _btrbk_date => $target_vol->{node}{BTRBK_DATE},
6685 };
6686
6687 # incomplete received (garbled) subvolumes have no received_uuid (as of btrfs-progs v4.3.1).
6688 # a subvolume in droot matching our naming is considered incomplete if received_uuid is not set!
6689 if($target_vol->{node}{received_uuid} eq '-') {
6690 $dstats{incomplete}++;
6691 $target_data->{status} = "incomplete";
6692 push @bdata, $target_data;
6693 next;
6694 }
6695
6696 foreach (@sdata) {
6697 if(_is_correlated($_->{_vinfo}{node}, $target_vol->{node})) {
6698 $target_data = {
6699 %$_,
6700 %$target_data,
6701 type => "snapshot,backup",
6702 _correlated => 1,
6703 };
6704 $_->{_correlated} = 1;
6705 last;
6706 }
6707 }
6708 push @bdata, $target_data;
6709 $latest_backup = $target_data if(!defined($latest_backup) || (cmp_date($latest_backup->{_btrbk_date}, $target_data->{_btrbk_date}) < 0));
6710
6711 $dstats{uptodate} ||= ($target_data->{status} // "") eq "up-to-date";
6712 $dstats{backups}++;
6713 if($target_data->{_correlated}) { $dstats{correlated}++; } else { $dstats{orphaned}++; }
6714 }
6715 push @ldata, $latest_backup;
6716
6717 push @stdata, {
6718 %svol_stats_data,
6719 %dstats,
6720 vinfo_prefixed_keys("target", $droot),
6721 target_subvolume => "$droot->{PATH}/$snapshot_name.*",
6722 backup_status => $dstats{uptodate} ? "up-to-date" : "",
6723 };
6724 $stats{$_} += $dstats{$_} foreach(qw(backups correlated incomplete orphaned));
6725 }
6726
6727 if($action_resolve eq "snapshots") {
6728 push @data, @sdata;
6729 } elsif($action_resolve eq "backups") {
6730 push @data, @bdata;
6731 } elsif($action_resolve eq "all") {
6732 push @data, sort { cmp_date($a->{_btrbk_date}, $b->{_btrbk_date}) } (@bdata, grep { !$_->{_correlated} } @sdata);
6733 } elsif($action_resolve eq "latest") {
6734 my $latest_snapshot = (sort { cmp_date($b->{_btrbk_date}, $a->{_btrbk_date}) } (@sdata, @bdata))[0];
6735 push @data, @ldata;
6736 push @data, $latest_snapshot if($latest_snapshot && !$latest_snapshot->{_correlated});
6737 } elsif($action_resolve eq "stats") {
6738 @stdata = ( \%svol_stats_data ) unless(@stdata);
6739 push @data, @stdata;
6740 }
6741 }
6742 }
6743
6744 if($action_resolve eq "stats") {
6745 my $filter = $config->{CMDLINE_FILTER_LIST} ? " (" . join(", ", @{$config->{CMDLINE_FILTER_LIST}}) . ")" : "";
6746 my @backup_total = map { $stats{$_} ? "$stats{$_} $_" : () } qw( correlated incomplete );
6747 my $bflags = @backup_total ? "(" . join(', ', @backup_total) . ")" : undef;
6748
6749 print_formatted("stats", \@data, paragraph => 1);
6750 print "Total${filter}:\n";
6751 print_formatted({ table => [ qw( a b -c ) ], RALIGN => { a=>1 } },
6752 [ { a => $stats{snapshots}, b => "snapshots" },
6753 { a => $stats{backups}, b => "backups", c => $bflags } ],
6754 output_format => "table", no_header => 1, empty_cell_char => "");
6755 }
6756 elsif($action_resolve eq "snapshots") {
6757 print_formatted("snapshots", \@data);
6758 }
6759 elsif($action_resolve eq "backups") {
6760 print_formatted("backups", \@data);
6761 }
6762 elsif($action_resolve eq "latest") {
6763 print_formatted("latest", \@data);
6764 }
6765 else {
6766 print_formatted("resolved", \@data);
6767 }
6768
6769 exit exit_status($config);
6770 }
6771
6772
6773 if($action_clean)
6774 {
6775 #
6776 # identify and delete incomplete backups
6777 #
6778 init_transaction_log(config_key($config, "transaction_log"),
6779 config_key($config, "transaction_syslog"));
6780
6781 my @out;
6782 foreach my $sroot (vinfo_subsection($config, 'volume')) {
6783 foreach my $svol (vinfo_subsection($sroot, 'subvolume')) {
6784 my $snapshot_name = config_key($svol, "snapshot_name") // die;
6785 foreach my $droot (vinfo_subsection($svol, 'target')) {
6786 INFO "Cleaning incomplete backups in: $droot->{PRINT}/$snapshot_name.*";
6787 push @out, "$droot->{PRINT}/$snapshot_name.*";
6788
6789 # incomplete received (garbled) subvolumes are not readonly and have no received_uuid (as of btrfs-progs v4.3.1).
6790 # a subvolume in droot matching our naming is considered incomplete if received_uuid is not set!
6791 my @delete = grep $_->{node}{received_uuid} eq '-', @{vinfo_subvol_list($droot, btrbk_direct_leaf => $snapshot_name, sort => 'path')};
6792 my @delete_success;
6793 foreach my $target_vol (@delete) {
6794 DEBUG "Found incomplete target subvolume: $target_vol->{PRINT}";
6795 if(btrfs_subvolume_delete($target_vol, commit => config_key($droot, "btrfs_commit_delete"), type => "delete_garbled")) {
6796 push(@delete_success, $target_vol);
6797 }
6798 }
6799
6800 INFO "Deleted " . scalar(@delete_success) . " incomplete backups in: $droot->{PRINT}/$snapshot_name.*";
6801 $droot->{SUBVOL_DELETED} //= [];
6802 push @{$droot->{SUBVOL_DELETED}}, @delete_success;
6803 push @out, map("--- $_->{PRINT}", @delete_success);
6804
6805 if(scalar(@delete_success) != scalar(@delete)) {
6806 ABORTED($droot, "Failed to delete incomplete target subvolume");
6807 push @out, "!!! Target \"$droot->{PRINT}\" aborted: " . ABORTED_TEXT($droot);
6808 }
6809 push(@out, "<no_action>") unless(scalar(@delete));
6810 push(@out, "");
6811 }
6812 }
6813 }
6814
6815 my $exit_status = exit_status($config);
6816 my $time_elapsed = time - $start_time;
6817 INFO "Completed within: ${time_elapsed}s (" . localtime(time) . ")";
6818 action("finished",
6819 status => $exit_status ? "partial" : "success",
6820 duration => $time_elapsed,
6821 message => $exit_status ? "At least one delete operation failed" : undef,
6822 );
6823 close_transaction_log();
6824
6825 #
6826 # print summary
6827 #
6828 unless($quiet)
6829 {
6830 $output_format ||= "custom";
6831 if($output_format eq "custom")
6832 {
6833 print_header(title => "Cleanup Summary",
6834 config => $config,
6835 time => $start_time,
6836 legend => [
6837 "--- deleted subvolume (incomplete backup)",
6838 ],
6839 );
6840 print join("\n", @out);
6841 print_footer($config, $exit_status);
6842 }
6843 else
6844 {
6845 # print action log (without transaction start messages)
6846 my @data = grep { $_->{status} !~ /starting$/ } @transaction_log;
6847 print_formatted("transaction", \@data, title => "TRANSACTION LOG");
6848 }
6849 }
6850
6851 exit $exit_status;
6852 }
6853
6854
6855 if($action_run)
6856 {
6857 init_transaction_log(config_key($config, "transaction_log"),
6858 config_key($config, "transaction_syslog"));
6859
6860 if($skip_snapshots) {
6861 INFO "Skipping snapshot creation (btrbk resume)";
6862 }
6863 else
6864 {
6865 #
6866 # create snapshots
6867 #
6868 foreach my $sroot (vinfo_subsection($config, 'volume')) {
6869 foreach my $svol (vinfo_subsection($sroot, 'subvolume')) {
6870 my $snaproot = vinfo_snapshot_root($svol);
6871 my $snapshot_basename = config_key($svol, "snapshot_name") // die;
6872 DEBUG "Evaluating snapshot creation for: $svol->{PRINT}";
6873
6874 # check if we need to create a snapshot
6875 my $snapshot_create = config_key($svol, "snapshot_create");
6876 if(not $snapshot_create) {
6877 DEBUG "Snapshot creation disabled (snapshot_create=no)";
6878 next;
6879 }
6880 elsif($snapshot_create eq "always") {
6881 DEBUG "Snapshot creation enabled (snapshot_create=always)";
6882 }
6883 elsif($snapshot_create eq "onchange") {
6884 # check if latest (btrbk only!) snapshot is up-to-date with source subvolume (by generation)
6885 my $latest = get_latest_related_snapshot($snaproot, $svol, $snapshot_basename);
6886 if($latest) {
6887 if($latest->{node}{cgen} == $svol->{node}{gen}) {
6888 INFO "Snapshot creation skipped: snapshot_create=onchange, snapshot is up-to-date: $latest->{PRINT}";
6889 $svol->{SNAPSHOT_UP_TO_DATE} = $latest;
6890 next;
6891 }
6892 DEBUG "Snapshot creation enabled: snapshot_create=onchange, gen=$svol->{node}{gen} > snapshot_cgen=$latest->{node}{cgen}";
6893 }
6894 else {
6895 DEBUG "Snapshot creation enabled: snapshot_create=onchange, no snapshots found";
6896 }
6897 }
6898 elsif($snapshot_create eq "ondemand") {
6899 # check if at least one target is present
6900 if(scalar vinfo_subsection($svol, 'target')) {
6901 DEBUG "Snapshot creation enabled (snapshot_create=ondemand): at least one target is present";
6902 }
6903 else {
6904 INFO "Snapshot creation skipped: snapshot_create=ondemand, and no target is present for: $svol->{PRINT}";
6905 next;
6906 }
6907 }
6908 else {
6909 die "illegal value for snapshot_create configuration option: $snapshot_create";
6910 }
6911
6912 # find unique snapshot name
6913 my $timestamp = timestamp(\@tm_now, config_key($svol, "timestamp_format"));
6914 my @unconfirmed_target_name;
6915 my @lookup = map { $_->{SUBVOL_PATH} } @{vinfo_subvol_list($snaproot)};
6916 foreach my $droot (vinfo_subsection($svol, 'target', 1)) {
6917 if(IS_ABORTED($droot)) {
6918 push(@unconfirmed_target_name, $droot);
6919 next;
6920 }
6921 push(@lookup, map { $_->{SUBVOL_PATH} } @{vinfo_subvol_list($droot)});
6922 }
6923 @lookup = grep /^\Q$snapshot_basename.$timestamp\E(_[0-9]+)?$/ ,@lookup;
6924 TRACE "Present snapshot names for \"$svol->{PRINT}\": " . join(', ', @lookup) if($do_trace);
6925 @lookup = map { /_([0-9]+)$/ ? $1 : 0 } @lookup;
6926 @lookup = sort { $b <=> $a } @lookup;
6927 my $postfix_counter = $lookup[0] // -1;
6928 $postfix_counter++;
6929 my $snapshot_name = $snapshot_basename . '.' . $timestamp . ($postfix_counter ? "_$postfix_counter" : "");
6930
6931 if(@unconfirmed_target_name) {
6932 INFO "Assuming non-present subvolume \"$snapshot_name\" in skipped targets: " . join(", ", map { "\"$_->{PRINT}\"" } @unconfirmed_target_name);
6933 }
6934
6935 # finally create the snapshot
6936 INFO "Creating subvolume snapshot for: $svol->{PRINT}";
6937 my $snapshot = vinfo_child($snaproot, "$snapshot_name");
6938 if(btrfs_subvolume_snapshot($svol, $snapshot))
6939 {
6940 vinfo_inject_child($snaproot, $snapshot, {
6941 parent_uuid => $svol->{node}{uuid},
6942 received_uuid => '-',
6943 readonly => 1,
6944 FORCE_PRESERVE => 'preserve forced: created just now',
6945 });
6946 $svol->{SNAPSHOT_CREATED} = $snapshot;
6947 }
6948 else {
6949 ABORTED($svol, "Failed to create snapshot: $svol->{PRINT} -> $snapshot->{PRINT}");
6950 WARN "Skipping subvolume section: " . ABORTED_TEXT($svol);
6951 }
6952 }
6953 }
6954 }
6955
6956 #
6957 # create backups
6958 #
6959 if($skip_backups) {
6960 INFO "Skipping backup creation (btrbk snapshot)";
6961 }
6962 else {
6963 foreach my $sroot (vinfo_subsection($config, 'volume')) {
6964 foreach my $svol (vinfo_subsection($sroot, 'subvolume')) {
6965 my $snaproot = vinfo_snapshot_root($svol);
6966 my $snapshot_basename = config_key($svol, "snapshot_name") // die;
6967 my @snapshots = sort({ cmp_date($a->{node}{BTRBK_DATE}, $b->{node}{BTRBK_DATE}) }
6968 @{vinfo_subvol_list($snaproot, readonly => 1, btrbk_direct_leaf => $snapshot_basename)});
6969 foreach my $droot (vinfo_subsection($svol, 'target')) {
6970 INFO "Checking for missing backups of subvolume \"$svol->{PRINT}\" in \"$droot->{PRINT}/\"";
6971 my @schedule;
6972 my $resume_total = 0;
6973 my $resume_success = 0;
6974
6975 foreach my $snapshot (@snapshots)
6976 {
6977 if(get_receive_targets($droot, $snapshot, exact => 1, warn => 1)){
6978 DEBUG "Found correlated target of: $snapshot->{PRINT}";
6979 next;
6980 }
6981
6982 DEBUG "Adding backup candidate: $snapshot->{PRINT}";
6983 push(@schedule, { value => $snapshot,
6984 btrbk_date => $snapshot->{node}{BTRBK_DATE},
6985 # not enforcing resuming of latest snapshot anymore (since v0.23.0)
6986 # preserve => $snapshot->{node}{FORCE_PRESERVE},
6987 });
6988 }
6989
6990 if(scalar @schedule)
6991 {
6992 DEBUG "Checking schedule for backup candidates";
6993 # add all present backups as informative_only: these are needed for correct results of schedule()
6994 foreach my $vol (@{vinfo_subvol_list($droot, btrbk_direct_leaf => $snapshot_basename)}) {
6995 push(@schedule, { informative_only => 1,
6996 value => $vol,
6997 btrbk_date => $vol->{node}{BTRBK_DATE},
6998 });
6999 }
7000 my ($preserve, undef) = schedule(
7001 schedule => \@schedule,
7002 preserve => config_preserve_hash($droot, "target"),
7003 );
7004 my @resume = grep defined, @$preserve; # remove entries with no value from list (target subvolumes)
7005 $resume_total = scalar @resume;
7006
7007 foreach my $snapshot (sort { $a->{node}{cgen} <=> $b->{node}{cgen} } @resume)
7008 {
7009 # Continue gracefully (skip instead of abort) on existing (possibly garbled) target
7010 if(my $err_vol = vinfo_subvol($droot, $snapshot->{NAME})) {
7011 my $err_msg = "Please delete stray subvolumes: \"btrbk clean $droot->{PRINT}\"";
7012 FIX_MANUALLY($droot, $err_msg);
7013 WARN "Target subvolume \"$err_vol->{PRINT}\" exists, but is not a receive target of \"$snapshot->{PRINT}\"";
7014 WARN $err_msg;
7015 WARN "Skipping backup of: $snapshot->{PRINT}";
7016 $droot->{SUBVOL_RECEIVED} //= [];
7017 push(@{$droot->{SUBVOL_RECEIVED}}, { ERROR => 1, received_subvolume => $err_vol });
7018 next;
7019 }
7020
7021 my ($clone_src, $target_parent_node);
7022 my $parent = get_best_parent($snapshot, $snaproot, $droot,
7023 strict_related => ((config_key($droot, "incremental") // "") eq "strict"),
7024 clone_src => \$clone_src,
7025 target_parent_node => \$target_parent_node);
7026 if(macro_send_receive(source => $snapshot,
7027 target => $droot,
7028 parent => $parent, # this is <undef> if no suitable parent found
7029 clone_src => $clone_src,
7030 target_parent_node => $target_parent_node,
7031 ))
7032 {
7033 $resume_success++;
7034 }
7035 else {
7036 # note: ABORTED flag is already set by macro_send_receive()
7037 ERROR("Error while resuming backups, aborting");
7038 last;
7039 }
7040 }
7041 }
7042
7043 if($resume_total) {
7044 INFO "Created $resume_success/$resume_total missing backups";
7045 } else {
7046 INFO "No missing backups found";
7047 }
7048 }
7049 }
7050 }
7051 }
7052
7053
7054 #
7055 # remove backups following a preserve daily/weekly/monthly scheme
7056 #
7057 my $schedule_results;
7058 if($preserve_snapshots && $preserve_backups) {
7059 INFO "Preserving all snapshots and backups";
7060 }
7061 else
7062 {
7063 $schedule_results = [];
7064 foreach my $sroot (vinfo_subsection($config, 'volume')) {
7065 foreach my $svol (vinfo_subsection($sroot, 'subvolume')) {
7066 my $snaproot = vinfo_snapshot_root($svol);
7067 my $snapshot_basename = config_key($svol, "snapshot_name") // die;
7068 my $target_aborted = 0;
7069 my @snapshots = sort({ cmp_date($b->{node}{BTRBK_DATE}, $a->{node}{BTRBK_DATE}) } # sort descending
7070 @{vinfo_subvol_list($snaproot, readonly => 1, btrbk_direct_leaf => $snapshot_basename)});
7071
7072 foreach my $droot (vinfo_subsection($svol, 'target', 1)) {
7073 if(IS_ABORTED($droot)) {
7074 if(IS_ABORTED($droot, "skip_cmdline_")) {
7075 $target_aborted ||= -1;
7076 } else {
7077 $target_aborted = 1;
7078 }
7079 next;
7080 }
7081
7082 # preserve latest common snapshot/backup (for incremental targets)
7083 if(config_key($droot, "incremental")) {
7084 foreach my $snapshot (@snapshots) {
7085 my @receive_targets = get_receive_targets($droot, $snapshot, exact => 1);
7086 if(scalar(@receive_targets)) {
7087 DEBUG "Force preserve for latest common snapshot: $snapshot->{PRINT}";
7088 $snapshot->{node}{FORCE_PRESERVE} = 'preserve forced: latest common snapshot';
7089 foreach(@receive_targets) {
7090 DEBUG "Force preserve for latest common target: $_->{PRINT}";
7091 $_->{node}{FORCE_PRESERVE} = 'preserve forced: latest common target';
7092 }
7093 last;
7094 }
7095 }
7096 }
7097
7098 if($preserve_backups) {
7099 INFO "Preserving all backups";
7100 }
7101 else {
7102 #
7103 # delete backups
7104 #
7105 INFO "Cleaning backups of subvolume \"$svol->{PRINT}\": $droot->{PRINT}/$snapshot_basename.*";
7106 unless(macro_delete($droot, $snapshot_basename, $droot,
7107 { preserve => config_preserve_hash($droot, "target"),
7108 results => $schedule_results,
7109 result_hints => { topic => "backup", root_path => $droot->{PATH} },
7110 },
7111 commit => config_key($droot, "btrfs_commit_delete"),
7112 type => "delete_target",
7113 qgroup => { destroy => config_key($droot, "target_qgroup_destroy"),
7114 type => "qgroup_destroy_target" },
7115 ))
7116 {
7117 $target_aborted = 1;
7118 }
7119 }
7120 }
7121
7122 #
7123 # delete snapshots
7124 #
7125 if($preserve_snapshots) {
7126 INFO "Preserving all snapshots";
7127 }
7128 elsif($target_aborted) {
7129 if($target_aborted == -1) {
7130 INFO "Skipping cleanup of snapshots for subvolume \"$svol->{PRINT}\", as at least one target is skipped by command line argument";
7131 } else {
7132 WARN "Skipping cleanup of snapshots for subvolume \"$svol->{PRINT}\", as at least one target aborted earlier";
7133 }
7134 }
7135 else {
7136 INFO "Cleaning snapshots" . ($wipe_snapshots ? " (wipe)" : "") . ": $snaproot->{PRINT}/$snapshot_basename.*";
7137 macro_delete($snaproot, $snapshot_basename, $svol,
7138 { preserve => config_preserve_hash($svol, "snapshot", wipe => $wipe_snapshots),
7139 results => $schedule_results,
7140 result_hints => { topic => "snapshot", root_path => $snaproot->{PATH} },
7141 },
7142 commit => config_key($svol, "btrfs_commit_delete"),
7143 type => "delete_snapshot",
7144 qgroup => { destroy => config_key($svol, "snapshot_qgroup_destroy"),
7145 type => "qgroup_destroy_snapshot" },
7146 );
7147 }
7148 }
7149 }
7150 }
7151
7152 my $exit_status = exit_status($config);
7153 my $time_elapsed = time - $start_time;
7154 INFO "Completed within: ${time_elapsed}s (" . localtime(time) . ")";
7155 action("finished",
7156 status => $exit_status ? "partial" : "success",
7157 duration => $time_elapsed,
7158 message => $exit_status ? "At least one backup task aborted" : undef,
7159 );
7160 close_transaction_log();
7161
7162
7163 unless($quiet)
7164 {
7165 #
7166 # print scheduling results
7167 #
7168 if($print_schedule && $schedule_results) {
7169 my @data = map { { %$_, vinfo_prefixed_keys("", $_->{value}) }; } @$schedule_results;
7170 my @data_snapshot = grep { $_->{topic} eq "snapshot" } @data;
7171 my @data_backup = grep { $_->{topic} eq "backup" } @data;
7172
7173 if(scalar(@data_snapshot)) {
7174 print_formatted("schedule", \@data_snapshot, title => "SNAPSHOT SCHEDULE", paragraph => 1);
7175 }
7176 if(scalar(@data_backup)) {
7177 print_formatted("schedule", \@data_backup, title => "BACKUP SCHEDULE", paragraph => 1);
7178 }
7179 }
7180
7181
7182 #
7183 # print summary
7184 #
7185 $output_format ||= "custom";
7186 if($output_format eq "custom")
7187 {
7188 my @out;
7189 foreach my $sroot (vinfo_subsection($config, 'volume', 1)) {
7190 foreach my $svol (vinfo_subsection($sroot, 'subvolume', 1)) {
7191 my @subvol_out;
7192 if($svol->{SNAPSHOT_UP_TO_DATE}) {
7193 push @subvol_out, "=== $svol->{SNAPSHOT_UP_TO_DATE}->{PRINT}";
7194 }
7195 if($svol->{SNAPSHOT_CREATED}) {
7196 push @subvol_out, "+++ $svol->{SNAPSHOT_CREATED}->{PRINT}";
7197 }
7198 foreach(@{$svol->{SUBVOL_DELETED} // []}) {
7199 push @subvol_out, "--- $_->{PRINT}";
7200 }
7201 foreach my $droot (vinfo_subsection($svol, 'target', 1)) {
7202 foreach(@{$droot->{SUBVOL_RECEIVED} // []}) {
7203 my $create_mode = "***";
7204 $create_mode = ">>>" if($_->{parent});
7205 # substr($create_mode, 0, 1, '%') if($_->{resume});
7206 $create_mode = "!!!" if($_->{ERROR});
7207 push @subvol_out, "$create_mode $_->{received_subvolume}->{PRINT}";
7208 }
7209
7210 foreach(@{$droot->{SUBVOL_DELETED} // []}) {
7211 push @subvol_out, "--- $_->{PRINT}";
7212 }
7213
7214 if(IS_ABORTED($droot, "abort_")) {
7215 push @subvol_out, "!!! Target \"$droot->{PRINT}\" aborted: " . ABORTED_TEXT($droot);
7216 }
7217 }
7218
7219 if(IS_ABORTED($sroot, "abort_")) {
7220 # repeat volume errors in subvolume context
7221 push @subvol_out, "!!! Volume \"$sroot->{PRINT}\" aborted: " . ABORTED_TEXT($sroot);
7222 }
7223 if(IS_ABORTED($svol, "abort_")) {
7224 # don't print "<no_action>" on skip_cmdline or skip_noauto
7225 push @subvol_out, "!!! Aborted: " . ABORTED_TEXT($svol);
7226 }
7227
7228 # print "<no_action>" for subvolume, unless aborted by "skip_"
7229 unless(scalar(@subvol_out) || IS_ABORTED($sroot, "skip_") || IS_ABORTED($svol, "skip_")) {
7230 @subvol_out = "<no_action>";
7231 }
7232
7233 if(@subvol_out) {
7234 push @out, "$svol->{PRINT}", @subvol_out, "";
7235 }
7236 }
7237 }
7238
7239 my @cmdline_options = map { "exclude: $_" } @exclude_cmdline;
7240 push @cmdline_options, "$skip_snapshots: No snapshots created" if($skip_snapshots);
7241 push @cmdline_options, "$skip_backups: No backups created" if($skip_backups);
7242 push @cmdline_options, "$preserve_snapshots: Preserved all snapshots" if($preserve_snapshots);
7243 push @cmdline_options, "$preserve_backups: Preserved all backups" if($preserve_backups);
7244
7245 print_header(title => "Backup Summary",
7246 config => $config,
7247 time => $start_time,
7248 options => \@cmdline_options,
7249 legend => [
7250 "=== up-to-date subvolume (source snapshot)",
7251 "+++ created subvolume (source snapshot)",
7252 "--- deleted subvolume",
7253 "*** received subvolume (non-incremental)",
7254 ">>> received subvolume (incremental)",
7255 ],
7256 );
7257
7258 print join("\n", @out);
7259 print_footer($config, $exit_status);
7260 }
7261 else
7262 {
7263 # print action log (without transaction start messages)
7264 my @data = grep { $_->{status} !~ /starting$/ } @transaction_log;
7265 print_formatted("transaction", \@data, title => "TRANSACTION LOG");
7266 }
7267 }
7268
7269 exit $exit_status if($exit_status);
7270 }
7271 }
7272
7273
7274 1;