| 1 |
#!/bin/sh |
| 2 |
# SPDX-FileCopyrightText: 2020-2026 Jonas Smedegaard <[email protected]> |
| 3 |
# |
| 4 |
# SPDX-FileCopyrightText: 2020-2021 Purism, SPC |
| 5 |
# |
| 6 |
# SPDX-License-Identifier: GPL-3.0-or-later |
| 7 |
# |
| 8 |
# Description: helper script to update copyright_hints |
| 9 |
# |
| 10 |
# Depends: |
| 11 |
# licensecheck, |
| 12 |
# libimage-exiftool-perl, |
| 13 |
# libipc-system-simple-perl, |
| 14 |
# libpath-tiny-perl, |
| 15 |
# libregexp-assemble-perl, |
| 16 |
# perl, |
| 17 |
|
| 18 |
set -eu |
| 19 |
|
| 20 |
EXT_skip='alaw mulaw mjr' |
| 21 |
RE_skip='fuzzers/corpora/.*' |
| 22 |
EXT_meta='jpeg jpg png mp4 opus webp' |
| 23 |
|
| 24 |
ext2re() { perl -sE 'printf ".*\.(%s)", join "|", split " ", $e' -- -e="$1"; } |
| 25 |
ext2opt() { perl -sE 'say map {" $o $_"} split " ", $e' -- -o="$1" -e="$2"; } |
| 26 |
|
| 27 |
RE_skip=$(ext2re "$EXT_skip")${RE_skip:+|$RE_skip} |
| 28 |
RE_meta=$(ext2re "$EXT_meta")${RE_meta:+|$RE_meta} |
| 29 |
RE_debian='debian/(changelog|copyright(_hints)?|source/lintian-overrides)' |
| 30 |
RE_hint='skip|meta' |
| 31 |
|
| 32 |
# cleanup stray hint files from a previous run |
| 33 |
find ./* -type f -regextype posix-egrep -regex "^\./.*:($RE_hint)$" -delete |
| 34 |
|
| 35 |
echo 'skip binary files without parsable metadata ...' 1>&2 |
| 36 |
find ./* -type f -regextype posix-egrep -regex "^\./($RE_skip)$" \ |
| 37 |
-exec sh -c 'echo "License: UNKNOWN" > "$1:skip"' shell {} ';' |
| 38 |
|
| 39 |
echo 'extract metadata from binary files ...' 1>&2 |
| 40 |
exiftool '-textOut!' %d%f.%e:meta -short -short -recurse -extractEmbedded \ |
| 41 |
$(ext2opt -ext "$EXT_meta") \ |
| 42 |
-- ./* |
| 43 |
|
| 44 |
# resolve file regex from contained license or shebang, or path regex |
| 45 |
_file_regex() { |
| 46 |
perl -Mv5.36 -MGetopt::Long=:config,gnu_getopt -MPath::Tiny -MRegexp::Assemble -MList::Util=any -- - "$@" <<'EOF' |
| 47 |
my (%opt, @license, @files, @match); |
| 48 |
GetOptions ( \%opt, "shortname=s@", "grantglob=s@", "regex=s@", "shebang=s@", "nonverb=s@" ); |
| 49 |
my @section = split /\n\n+/, path("debian/copyright")->slurp_utf8; |
| 50 |
for my $name ( @{ $opt{shortname} // [] } ) { |
| 51 |
push @license, map {/^License:\h*\Q$name\E\n\h+(\S[^\n]*(?:\n\h+\S[^\n]*)*)/} @section; } |
| 52 |
for my $glob ( @{ $opt{grantglob} // [] } ) { |
| 53 |
push @files, grep {/^Files:\n?\h(?:\S[^\n]*\n?\h)*\Q$glob\E\s/} @section; } |
| 54 |
my @grant = map {/^License(?:-Grant:|:\h*\S[^\n]*)\h*\n\h+(\S[^\n]*(?:\n\h+\S[^\n]*)*)/mg} @files; |
| 55 |
my @firstline_re = map {qr/^\Q$_\E/} @{ $opt{shebang} // [] }; |
| 56 |
my $nonverb_re = Regexp::Assemble->new |
| 57 |
->add("\\W+", map {"\\W+(?:$_)\\W+"} @{ $opt{nonverb} // [] }) |
| 58 |
->as_string; |
| 59 |
my @content_re = map { |
| 60 |
s/\W+/[**]/g; |
| 61 |
s/\Q[**]\E\d\Q[**]\E/[**]\\S{0,2}[**]/g; |
| 62 |
s/\Q[**]\E/$nonverb_re/g; |
| 63 |
qr/$_/ |
| 64 |
} @license, @grant; |
| 65 |
my $inspect = sub { |
| 66 |
return if $_[0]->is_dir; |
| 67 |
if (@firstline_re) { |
| 68 |
my ($head) = $_[0]->lines({ count => 1 }); |
| 69 |
push @match, quotemeta($_[0]) and return |
| 70 |
if any { $head =~ $_ } @firstline_re; |
| 71 |
} |
| 72 |
push @match, quotemeta($_[0]) |
| 73 |
if any { $_[0]->slurp_raw =~ $_ } @content_re; |
| 74 |
}; |
| 75 |
for (@ARGV) { |
| 76 |
my $p = path($_); |
| 77 |
$p->is_dir ? $p->visit($inspect, { recurse => 1 }) : $inspect->($p); |
| 78 |
} |
| 79 |
my $files_re = Regexp::Assemble->new->add(@match, @{ $opt{regex} // [] }); |
| 80 |
print $files_re->as_string =~ s/\(\?:/\(/gr; |
| 81 |
EOF |
| 82 |
} |
| 83 |
|
| 84 |
RE_SKIP="$RE_skip|$RE_meta" |
| 85 |
|
| 86 |
# major licensing patterns |
| 87 |
RE_janus=$(_file_regex --grantglob '*' -- *) |
| 88 |
|
| 89 |
# TODO: automate more of this manual cleanup: |
| 90 |
# * strip garbage copyright holders |
| 91 |
# * optionally merge equally licensed Files sections |
| 92 |
# * do "sort -k2 -k1,1 -u" on copyright holders |
| 93 |
# * merge copyright years for each copyright holder |
| 94 |
# TODO: strip files matching glob in current (only, no later) section |
| 95 |
_licensecheck() { |
| 96 |
perl -Mv5.36 -MGetopt::Long=:config,gnu_getopt \ |
| 97 |
-MIPC::System::Simple=capture -MList::Util=uniq \ |
| 98 |
-- - --hint="$RE_hint" "$@" <<'EOF' >> debian/copyright_hints |
| 99 |
my %opt; |
| 100 |
GetOptions ( \%opt, "merge-licenses", |
| 101 |
"hint=s", "check=s", "ignore=s", "shortname=s", "subset=s" ); |
| 102 |
my @subset = exists $opt{subset} ? split(" ", $opt{subset}) : (); |
| 103 |
my $subset_globs = join( "\n ", @subset ); |
| 104 |
if ( $subset_globs =~ /^[*]$/ ) { say STDERR "check default section(s) ..." } |
| 105 |
elsif ( @subset and $opt{shortname} ) { say STDERR "check $opt{shortname} section(s) @subset ..." } |
| 106 |
elsif (@subset) { say STDERR "check section(s) @subset ..." } |
| 107 |
elsif ( $opt{shortname} ) { say STDERR "check $opt{shortname} section(s) ..." } |
| 108 |
else { say STDERR "check remaining upstream section(s) ..." } |
| 109 |
my @cmd = ( qw(licensecheck --copyright --deb-machine --recursive --lines 0), |
| 110 |
"--check", $opt{check}, "--ignore", $opt{ignore}, |
| 111 |
($opt{"merge-licenses"} ? "--merge-licenses" : ()), "--" ); |
| 112 |
say STDERR "@cmd *" if $ENV{DEBUG}; |
| 113 |
local $_ = capture( @cmd, glob "*" ); |
| 114 |
if ( !$ENV{NOGLOBMERGE} and grep /[*]/, @subset ) { s/^.*?\n\nFiles: \K.*?(?=\n\w)/$subset_globs/s } |
| 115 |
elsif (@subset) { s/^.*?\n\nFiles: \K/$subset_globs\n /s } |
| 116 |
s/^.*?\n\n//s if exists $opt{subset} and (!@subset or $subset[0] ne "*"); |
| 117 |
s/^Files:\K /\n /mg; |
| 118 |
s/^[C]opyright:\K /\n /mg; |
| 119 |
s/(?:(?<=^ )|(?<=\d{4})),\K (?=\d{4})//mg; |
| 120 |
s/:(?:$opt{hint})$//mg if $opt{hint}; |
| 121 |
if ($opt{shortname}) { s/^License: \K(.*)/ join " and\/or ", uniq sort grep( !m{\AUNKNOWN\Z}, split(" and\/or ", $1), $opt{shortname} ) /mge } |
| 122 |
print; |
| 123 |
EOF |
| 124 |
} |
| 125 |
|
| 126 |
rm -f debian/copyright_hints |
| 127 |
|
| 128 |
# initially, check all to know roughly what to group and in which order |
| 129 |
#rm -f debian/copyright_hints |
| 130 |
#_licensecheck '' --check '.*' --ignore "^($RE_SKIP|debian/.*)$" |
| 131 |
#exit 0 |
| 132 |
|
| 133 |
# check default licensed files first |
| 134 |
_licensecheck --shortname 'GPL-3 with OpenSSL exception' --subset '*' --check "^($RE_janus)$" --ignore "^($RE_SKIP|debian/.*)$" |
| 135 |
|
| 136 |
# check generally |
| 137 |
# * omit non-copyright-protected Debian files |
| 138 |
_licensecheck --subset '' --check '.*' --ignore "^($RE_janus|$RE_SKIP|debian/.*)$" |
| 139 |
_licensecheck --subset 'debian/*' --check '^debian/' --ignore "^($RE_janus|$RE_SKIP|$RE_debian)$" |
| 140 |
|
| 141 |
# cleanup hint files |
| 142 |
find ./* -type f -regextype posix-egrep -regex "^\./.*:($RE_hint)$" -delete |