Debian Codegraph @

debian/copyright-check

raw
1 #!/bin/sh
2 # SPDX-FileCopyrightText: 2020-2026 Jonas Smedegaard <[email protected]>
3 #
4 # SPDX-FileCopyrightText: 2020-2021 Purism, SPC
5 #
6 # SPDX-License-Identifier: GPL-3.0-or-later
7 #
8 # Description: helper script to update copyright_hints
9 #
10 # Depends:
11 # licensecheck,
12 # libimage-exiftool-perl,
13 # libipc-system-simple-perl,
14 # libpath-tiny-perl,
15 # libregexp-assemble-perl,
16 # perl,
17
18 set -eu
19
20 EXT_skip='alaw mulaw mjr'
21 RE_skip='fuzzers/corpora/.*'
22 EXT_meta='jpeg jpg png mp4 opus webp'
23
24 ext2re() { perl -sE 'printf ".*\.(%s)", join "|", split " ", $e' -- -e="$1"; }
25 ext2opt() { perl -sE 'say map {" $o $_"} split " ", $e' -- -o="$1" -e="$2"; }
26
27 RE_skip=$(ext2re "$EXT_skip")${RE_skip:+|$RE_skip}
28 RE_meta=$(ext2re "$EXT_meta")${RE_meta:+|$RE_meta}
29 RE_debian='debian/(changelog|copyright(_hints)?|source/lintian-overrides)'
30 RE_hint='skip|meta'
31
32 # cleanup stray hint files from a previous run
33 find ./* -type f -regextype posix-egrep -regex "^\./.*:($RE_hint)$" -delete
34
35 echo 'skip binary files without parsable metadata ...' 1>&2
36 find ./* -type f -regextype posix-egrep -regex "^\./($RE_skip)$" \
37 -exec sh -c 'echo "License: UNKNOWN" > "$1:skip"' shell {} ';'
38
39 echo 'extract metadata from binary files ...' 1>&2
40 exiftool '-textOut!' %d%f.%e:meta -short -short -recurse -extractEmbedded \
41 $(ext2opt -ext "$EXT_meta") \
42 -- ./*
43
44 # resolve file regex from contained license or shebang, or path regex
45 _file_regex() {
46 perl -Mv5.36 -MGetopt::Long=:config,gnu_getopt -MPath::Tiny -MRegexp::Assemble -MList::Util=any -- - "$@" <<'EOF'
47 my (%opt, @license, @files, @match);
48 GetOptions ( \%opt, "shortname=s@", "grantglob=s@", "regex=s@", "shebang=s@", "nonverb=s@" );
49 my @section = split /\n\n+/, path("debian/copyright")->slurp_utf8;
50 for my $name ( @{ $opt{shortname} // [] } ) {
51 push @license, map {/^License:\h*\Q$name\E\n\h+(\S[^\n]*(?:\n\h+\S[^\n]*)*)/} @section; }
52 for my $glob ( @{ $opt{grantglob} // [] } ) {
53 push @files, grep {/^Files:\n?\h(?:\S[^\n]*\n?\h)*\Q$glob\E\s/} @section; }
54 my @grant = map {/^License(?:-Grant:|:\h*\S[^\n]*)\h*\n\h+(\S[^\n]*(?:\n\h+\S[^\n]*)*)/mg} @files;
55 my @firstline_re = map {qr/^\Q$_\E/} @{ $opt{shebang} // [] };
56 my $nonverb_re = Regexp::Assemble->new
57 ->add("\\W+", map {"\\W+(?:$_)\\W+"} @{ $opt{nonverb} // [] })
58 ->as_string;
59 my @content_re = map {
60 s/\W+/[**]/g;
61 s/\Q[**]\E\d\Q[**]\E/[**]\\S{0,2}[**]/g;
62 s/\Q[**]\E/$nonverb_re/g;
63 qr/$_/
64 } @license, @grant;
65 my $inspect = sub {
66 return if $_[0]->is_dir;
67 if (@firstline_re) {
68 my ($head) = $_[0]->lines({ count => 1 });
69 push @match, quotemeta($_[0]) and return
70 if any { $head =~ $_ } @firstline_re;
71 }
72 push @match, quotemeta($_[0])
73 if any { $_[0]->slurp_raw =~ $_ } @content_re;
74 };
75 for (@ARGV) {
76 my $p = path($_);
77 $p->is_dir ? $p->visit($inspect, { recurse => 1 }) : $inspect->($p);
78 }
79 my $files_re = Regexp::Assemble->new->add(@match, @{ $opt{regex} // [] });
80 print $files_re->as_string =~ s/\(\?:/\(/gr;
81 EOF
82 }
83
84 RE_SKIP="$RE_skip|$RE_meta"
85
86 # major licensing patterns
87 RE_janus=$(_file_regex --grantglob '*' -- *)
88
89 # TODO: automate more of this manual cleanup:
90 # * strip garbage copyright holders
91 # * optionally merge equally licensed Files sections
92 # * do "sort -k2 -k1,1 -u" on copyright holders
93 # * merge copyright years for each copyright holder
94 # TODO: strip files matching glob in current (only, no later) section
95 _licensecheck() {
96 perl -Mv5.36 -MGetopt::Long=:config,gnu_getopt \
97 -MIPC::System::Simple=capture -MList::Util=uniq \
98 -- - --hint="$RE_hint" "$@" <<'EOF' >> debian/copyright_hints
99 my %opt;
100 GetOptions ( \%opt, "merge-licenses",
101 "hint=s", "check=s", "ignore=s", "shortname=s", "subset=s" );
102 my @subset = exists $opt{subset} ? split(" ", $opt{subset}) : ();
103 my $subset_globs = join( "\n ", @subset );
104 if ( $subset_globs =~ /^[*]$/ ) { say STDERR "check default section(s) ..." }
105 elsif ( @subset and $opt{shortname} ) { say STDERR "check $opt{shortname} section(s) @subset ..." }
106 elsif (@subset) { say STDERR "check section(s) @subset ..." }
107 elsif ( $opt{shortname} ) { say STDERR "check $opt{shortname} section(s) ..." }
108 else { say STDERR "check remaining upstream section(s) ..." }
109 my @cmd = ( qw(licensecheck --copyright --deb-machine --recursive --lines 0),
110 "--check", $opt{check}, "--ignore", $opt{ignore},
111 ($opt{"merge-licenses"} ? "--merge-licenses" : ()), "--" );
112 say STDERR "@cmd *" if $ENV{DEBUG};
113 local $_ = capture( @cmd, glob "*" );
114 if ( !$ENV{NOGLOBMERGE} and grep /[*]/, @subset ) { s/^.*?\n\nFiles: \K.*?(?=\n\w)/$subset_globs/s }
115 elsif (@subset) { s/^.*?\n\nFiles: \K/$subset_globs\n /s }
116 s/^.*?\n\n//s if exists $opt{subset} and (!@subset or $subset[0] ne "*");
117 s/^Files:\K /\n /mg;
118 s/^[C]opyright:\K /\n /mg;
119 s/(?:(?<=^ )|(?<=\d{4})),\K (?=\d{4})//mg;
120 s/:(?:$opt{hint})$//mg if $opt{hint};
121 if ($opt{shortname}) { s/^License: \K(.*)/ join " and\/or ", uniq sort grep( !m{\AUNKNOWN\Z}, split(" and\/or ", $1), $opt{shortname} ) /mge }
122 print;
123 EOF
124 }
125
126 rm -f debian/copyright_hints
127
128 # initially, check all to know roughly what to group and in which order
129 #rm -f debian/copyright_hints
130 #_licensecheck '' --check '.*' --ignore "^($RE_SKIP|debian/.*)$"
131 #exit 0
132
133 # check default licensed files first
134 _licensecheck --shortname 'GPL-3 with OpenSSL exception' --subset '*' --check "^($RE_janus)$" --ignore "^($RE_SKIP|debian/.*)$"
135
136 # check generally
137 # * omit non-copyright-protected Debian files
138 _licensecheck --subset '' --check '.*' --ignore "^($RE_janus|$RE_SKIP|debian/.*)$"
139 _licensecheck --subset 'debian/*' --check '^debian/' --ignore "^($RE_janus|$RE_SKIP|$RE_debian)$"
140
141 # cleanup hint files
142 find ./* -type f -regextype posix-egrep -regex "^\./.*:($RE_hint)$" -delete