Debian Codegraph @

debian/dh-sphinxdoc/dh_sphinxdoc

raw
1 #!/usr/bin/perl
2
3 # Copyright © 2011 Jakub Wilk <[email protected]>
4 # © 2014-2026 Dmitry Shachnev <[email protected]>
5 #
6 # Redistribution and use in source and binary forms, with or without
7 # modification, are permitted provided that the following conditions are
8 # met:
9 #
10 # * Redistributions of source code must retain the above copyright
11 # notice, this list of conditions and the following disclaimer.
12 #
13 # * Redistributions in binary form must reproduce the above copyright
14 # notice, this list of conditions and the following disclaimer in the
15 # documentation and/or other materials provided with the distribution.
16 #
17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28
29 =head1 NAME
30
31 dh_sphinxdoc - helps with packaging Sphinx-generated documentation
32
33 =head1 SYNOPSIS
34
35 dh_sphinxdoc [S<I<debhelper options>>] [B<-X>I<item>] [I<directory>...]
36
37 =head1 DESCRIPTION
38
39 B<dh_sphinxdoc> is a debhelper program that prepares Sphinx-generated
40 documentation for inclusion in a Debian package. More specifically:
41
42 =over 4
43
44 =item *
45
46 It checks if all the files referenced by F<searchindex.js> exist.
47
48 =item *
49
50 It replaces known F<*.js> files with symlinks to F</usr/share/javascript/sphinxdoc/>
51 and generates B<${sphinxdoc:Depends}> substitution variable.
52
53 =item *
54
55 If the Sphinx RTD theme is used, it replaces known files from this theme with
56 symlinks to F</usr/share/sphinx_rtd_theme/>, and adds B<sphinx-rtd-theme-common>
57 to B<${sphinxdoc:Depends}>.
58
59 =item *
60
61 It provides a B<${sphinxdoc:Built-Using}> substitution variable, for tracking
62 files which could not be symlinked. Examples of such files are F<*.js> and F<*.css>
63 files that are generated from corresponding F<*.js.jinja> and F<*.css.jinja> templates,
64 and can vary depending on the used theme options (for instance, F<basic.css> file
65 is generated from F<basic.css.jinja> and is included in almost every Sphinx-generated
66 documentation). Currently, this variable will contain B<sphinx> and, if the default
67 theme is used, B<alabaster>, with their versions (other themes are not supported).
68
69 B<Note>: in most cases you do not need this variable. Use it only if that is required
70 by your license. See L<https://bugs.debian.org/1136263> for details.
71
72 =item *
73
74 It removes the F<.doctrees> directory.
75
76 =item *
77
78 It removes the F<.buildinfo> file.
79
80 =item *
81
82 It removes the F<websupport.js> file and F<*-stemmer.js> files.
83
84 =back
85
86 Note that B<dh_sphinxdoc> does not build the documentation, it only performs
87 the above operations when the documentation is already installed into the
88 package build path. To build the docs, please use L<sphinx-build(1)> command
89 or B<python3 -m sphinx> syntax.
90
91 You can pass B<--with sphinxdoc> to L<dh(1)> to make it automatically call
92 B<dh_sphinxdoc> after B<dh_installdocs>.
93
94 =head1 OPTIONS
95
96 =over 4
97
98 =item I<directory>
99
100 By default, B<dh_sphinxdoc> scans your package looking for directories looking
101 like they contain Sphinx-generated documentation. However, if you explicitly
102 provide one or more directories, only they will be processed. If documentation
103 is not found at I<directory>, an error is raised.
104
105 =item B<-X>I<item>, B<--exclude=>I<item>
106
107 Exclude files that contain I<item> anywhere in their filename from
108 being symlinked, removed or checked for existence.
109
110 =back
111
112 =head1 BUGS
113
114 Symlinking translations.js is not supported.
115
116 =cut
117
118 use strict;
119 use warnings;
120
121 use Digest::MD5;
122 use File::Basename;
123 use File::Find;
124 use Debian::Debhelper::Dh_Lib;
125 use JSON;
126
127 my %packaged_js = ();
128 my @cruft_js = qw(websupport.js);
129
130 sub md5($)
131 {
132 my ($filename) = @_;
133 my $md5 = Digest::MD5->new;
134 open(F, '<', $filename) or error("cannot open $filename");
135 $md5->addfile(*F);
136 close(F);
137 return $md5->digest;
138 }
139
140 sub load_packaged_js()
141 {
142 my %versions = ();
143 my $root = 'debian/libjs-sphinxdoc'; # It's tempting to use
144 # tmpdir('libjs-sphinxdoc') here, but it would break if the user passed
145 # --tmpdir to the command.
146 $root = '' unless -d $root;
147 my $path = "$root/usr/share/javascript/sphinxdoc";
148 open(F, '<', "$path/index") or error("cannot open $path/index");
149 while (<F>)
150 {
151 chomp;
152 next if /^(#|\s*$)/;
153 my ($js, $minver) = split(/\s+/, $_, 2);
154 unless (defined($minver))
155 {
156 $js =~ m{^([0-9.]+)/} or error("syntax error in $path/index");
157 $minver = $1;
158 }
159 $versions{$js} = $minver;
160 }
161 close(F);
162 find({
163 wanted => sub {
164 my $js = $_;
165 my ($jsbase, $jsname) = m{([0-9.]+/(\S+[.]js))$} or return;
166 my $version = $versions{$jsbase};
167 defined($version) or error("$jsbase is not in the index; is it up-to-date?");
168 delete $versions{$jsbase};
169 my $md5;
170 if (-l $js)
171 {
172 # Follow the symlink, but only if points *outside* our own directory.
173 my $js_target = readlink($js);
174 $js_target =~ m{^(/|\Q../../\E)} or return;
175 unless ($js_target =~ m{^/})
176 {
177 $js_target = "$js/../$js_target";
178 while ($js_target =~ s{[^./][^/]+/[.][.]/}{}) {};
179 }
180 $md5 = md5($js_target);
181 }
182 else
183 {
184 $js =~ s{^\Q$root\E}{} unless -f $js;
185 $md5 = md5($js);
186 }
187 $js =~ s{^\Q$root\E}{};
188 my $data = [$js, "libjs-sphinxdoc (>= $version)"];
189 $packaged_js{$md5} = $data;
190 # language_data.js added only by hash, because that file has the
191 # same name but different content for all supported languages.
192 unless ($jsname =~ m{language_data.*\.js}) {
193 $packaged_js{$jsname} = $data;
194 }
195 },
196 no_chdir => 1
197 }, $path);
198 map { error("$path/$_ is missing") } keys(%versions);
199 my %legacy_dependencies = (
200 "/usr/share/javascript/jquery/jquery.js" => "libjs-jquery (>= 3.6.0)",
201 "/usr/share/javascript/jquery/jquery.min.js" => "libjs-jquery (>= 3.6.0)",
202 "/usr/share/javascript/underscore/underscore.js" => "libjs-underscore (>= 1.3.1)",
203 "/usr/share/javascript/underscore/underscore.min.js" => "libjs-underscore (>= 1.3.1)",
204 );
205 while (my ($jsname, $dependency) = each %legacy_dependencies)
206 {
207 if (-e $jsname) {
208 $packaged_js{md5($jsname)} = [$jsname, $dependency];
209 }
210 $packaged_js{basename($jsname)} = [$jsname, $dependency];
211 }
212 }
213
214 sub looks_like_sphinx_doc($)
215 {
216 my ($path) = @_;
217 return 0 unless -f "$path/searchindex.js";
218 return 0 unless -f "$path/search.html";
219 return 1;
220 }
221
222 sub looks_like_sphinx_singlehtml_doc($)
223 {
224 my ($path) = @_;
225 return 0 unless -d "$path/_static";
226 return 0 if -f "$path/searchindex.js";
227
228 # There should be exactly one HTML file in singlehtml build.
229 my @html_files = glob("$path/*.html");
230 my @sphinx_html_files;
231 foreach my $html_file (@html_files)
232 {
233 open(my $fh, '<', $html_file) or error("cannot open $html_file");
234 while (my $line = <$fh>)
235 {
236 if ($line =~ /<script(?: type="text\/javascript")? src="_static\/doctools.js">/)
237 {
238 push @sphinx_html_files, $html_file;
239 last;
240 }
241 }
242 }
243 return 0 if @sphinx_html_files != 1;
244 return $sphinx_html_files[0];
245 }
246
247 sub sanity_check($$)
248 {
249 local $/;
250 my ($path, $singlehtml_file) = @_;
251 my $searchfn;
252 my $index;
253 if ($singlehtml_file)
254 {
255 # There is no search.html in singlehtml build, so we take the main HTML
256 # file for sanity checking and retrieving JS files.
257 $searchfn = $singlehtml_file;
258 }
259 else
260 {
261 my $indexfn = "$path/searchindex.js";
262 open(F, '<', $indexfn) or error("cannot open $indexfn");
263 $index = <F>;
264 close(F);
265 $index =~ m{^Search[.]setIndex[(](.*)[)]$} or error("$indexfn doesn't look like a Sphinx search index");
266 $index = decode_json($1);
267 $searchfn = "$path/search.html";
268 }
269 open(F, '<', $searchfn) or error("cannot open $searchfn");
270 my $search = <F>;
271 close F;
272 $search =~ s/<!--.*?-->//g; # strip comments
273 my %js = ();
274 grep {
275 s/[?#].*//;
276 s/\s+$//;
277 $js{$_} = 1 unless m/^[a-z][a-z0-9.+-]*:/i or excludefile("$path/$_");
278 } $search =~ m{<script(?: type="text/javascript")? src="([^"]++)"></script>}g;
279 my $documentation_options;
280 for my $line (split /^/, $search)
281 {
282 if ($line =~ "_static/documentation_options.js")
283 {
284 my $documentation_options_fn = "$path/_static/documentation_options.js";
285 open(my $fh, '<', $documentation_options_fn) or error("cannot open $documentation_options_fn");
286 $documentation_options = <$fh>;
287 close $fh;
288 }
289 if ($line =~ "var DOCUMENTATION_OPTIONS =")
290 {
291 $documentation_options = $search;
292 }
293 }
294 defined $documentation_options or error("DOCUMENTATION_OPTIONS not found");
295 my $loads_searchindex = $search =~ m{<script(?: type="text/javascript")? src="[^"]*searchindex.js\s?"(?: defer(?:="defer")?)?>};
296 unless ($loads_searchindex)
297 {
298 # old style, used before Sphinx 2.0 (but still used by sphinx-rtd-theme)
299 $loads_searchindex = $search =~ m/\QjQuery(function() { Search.loadIndex("\E/;
300 }
301 my ($has_source) = $documentation_options =~ m{HAS_SOURCE:\s*(true|false)};
302 my ($sourcelink_suffix) = $documentation_options =~ m{SOURCELINK_SUFFIX:\s*'([^']*)'};
303 $sourcelink_suffix = ".txt" unless defined $sourcelink_suffix;
304 my ($content_root) = $search =~ m{data-content_root="([^"]*)"};
305 unless (defined $content_root)
306 {
307 # Support the way zzzeeksphinx is setting it.
308 ($content_root) = $search =~ m{document\.documentElement\.dataset\.content_root = '([^']*)';};
309 }
310 unless (defined $content_root)
311 {
312 # We support fallback options using support_old_search_indexes.diff.
313 # Drop this block when that patch gets removed.
314 if ($documentation_options =~ /\QURL_ROOT: document.getElementById("documentation_options")\E/)
315 {
316 ($content_root) = $search =~ m{data-url_root="([^"]*)"};
317 }
318 else
319 {
320 ($content_root) = $documentation_options =~ m{URL_ROOT:\s*'([^']*)'};
321 }
322 }
323 %js or error("$searchfn does not include any JavaScript code");
324 $singlehtml_file or $loads_searchindex or error("$searchfn does not load searchindex.js");
325 defined $has_source or error("DOCUMENTATION_OPTIONS does not define HAS_SOURCE");
326 defined $content_root or error("$searchfn top-level node does not have data-content_root attribute");
327 $has_source = $has_source eq 'true';
328 $content_root =~ m{^([a-z]+:/)?/} and error("content_root in $searchfn is not relative");
329 for my $js (keys(%js))
330 {
331 -f "$path/$js" or -l "$path/$js" or error("$path/$js is missing");
332 }
333 unless ($singlehtml_file)
334 {
335 my $pages = $index->{"filenames"};
336 for my $page (@$pages)
337 {
338 # Append sourcelink_suffix if the page name does not already end with it.
339 (my $sourcepage = $page) =~ s/(?<!$sourcelink_suffix)$/$sourcelink_suffix/;
340 -f "$path/_sources/$sourcepage"
341 or excludefile("$path/_sources/$sourcepage")
342 or error("$path/_sources/$sourcepage is missing")
343 if $has_source;
344 # Get the page basename before appending .html.
345 $page =~ s/\.[a-z]+$//;
346 -f "$path/$page.html"
347 or excludefile("$path/$page.html")
348 or error("$path/$page.html is missing");
349 }
350 }
351 if (opendir(D, "$path/_static/"))
352 {
353 grep {
354 $js{"_static/$_"} = 1
355 if /[.]js$/ and not excludefile("$path/_static/$_");
356 } readdir(D);
357 closedir(D);
358 }
359 return keys(%js);
360 }
361
362 sub unknown_javascript($)
363 {
364 my ($js) = @_;
365 my $message = "unknown JavaScript code: $js";
366 $js =~ s{.*/}{};
367 my $basic = grep { $_ eq $js } qw(searchtools.js doctools.js);
368 my $cruft = grep { $_ eq $js } @cruft_js;
369 my $stemmer = $js =~ m{.*-stemmer\.js};
370 my @ignored_files = (
371 "documentation_options.js",
372 "searchindex.js",
373 "sidebar.js",
374 "theme.js",
375 # _sphinx_javascript_frameworks_compat.js is shipped in python3-sphinxcontrib.jquery.
376 # That package brings dependency on python3-sphinx itself, which we don't want in
377 # documentation packages. Given it's just 4.2 KB, ignore it.
378 "_sphinx_javascript_frameworks_compat.js",
379 );
380 my $basic_ignored = grep { $_ eq $js } @ignored_files;
381 if ($basic)
382 {
383 error("error: $message");
384 }
385 elsif (not $cruft and not $stemmer and not $basic_ignored)
386 {
387 warning("ignoring $message");
388 }
389 }
390
391 sub ln_sf($$)
392 {
393 my ($orig_target, $orig_source) = my ($target, $source) = @_;
394 $source =~ s{^debian/[^/]++/+}{} or die;
395 $target =~ s{^/++}{} or die;
396 my @source = split(m{/++}, $source);
397 my @target = split(m{/++}, $target);
398 @source > 0 and @target > 0 or die;
399 if ($source[0] eq $target[0])
400 {
401 # Make the symlink relative, as per Policy 10.5.
402 while (@source > 0 and @target > 0 and $source[0] eq $target[0])
403 {
404 shift @source;
405 shift @target;
406 }
407 $target = ('../' x $#source) . join('/', @target);
408 }
409 else
410 {
411 # Keep the symlink absolute, as per Policy 10.5.
412 $target = $orig_target;
413 }
414 doit('ln', '-sf', $target, $orig_source);
415 }
416
417 sub fix_symlinks($@)
418 {
419 my %deps = ();
420 my ($path, @js) = @_;
421 for my $js (@js)
422 {
423 my $id = '';
424 if (-l "$path/$js")
425 {
426 my $symlink_target = readlink("$path/$js");
427 $symlink_target =~ m{/sphinxdoc/} and next;
428 $symlink_target =~ m{/javascript/\w+/(\w+)([.](min|lite|pack))?[.]js$} and $id = "$1.js";
429 }
430 elsif (-f "$path/$js")
431 {
432 $id = md5("$path/$js");
433 }
434 if (exists $packaged_js{$id})
435 {
436 my ($target, $dependency) = @{$packaged_js{$id}};
437 ln_sf($target, "$path/$js");
438 $deps{$dependency} = 1;
439 }
440 else
441 {
442 unknown_javascript("$path/$js");
443 }
444 }
445 return keys %deps;
446 }
447
448 sub drop_cruft($)
449 {
450 my ($path) = @_;
451 my $doctrees = "$path/.doctrees";
452 my $buildinfo = "$path/.buildinfo";
453 if (-d $doctrees and not excludefile($doctrees))
454 {
455 doit('rm', '-rf', $doctrees);
456 }
457 if (-f $buildinfo and not excludefile($buildinfo))
458 {
459 doit('rm', '-f', $buildinfo);
460 }
461 foreach my $js (@cruft_js)
462 {
463 my $js = "$path/_static/$js";
464 if (-f $js and not excludefile($js))
465 {
466 doit('rm', '-f', $js) if -f $js;
467 }
468 }
469 while (my $stemmer_file = glob("$path/_static/*-stemmer.js"))
470 {
471 doit('rm', '-f', $stemmer_file);
472 }
473 }
474
475 sub process_rtd($)
476 {
477 my ($path) = @_;
478 my $theme_is_rtd = 0;
479 if (-d "$path/_static/js" and -f "$path/_static/js/theme.js")
480 {
481 if (open(F, '<', "$path/_static/js/theme.js"))
482 {
483 while (my $line = <F>) {
484 if (index($line, "window.SphinxRtdTheme") != -1)
485 {
486 $theme_is_rtd = 1;
487 last;
488 }
489 }
490 close(F);
491 }
492 }
493
494 my @deps;
495 my $target_dir = "/usr/share/sphinx_rtd_theme/static";
496 if ($theme_is_rtd and -d $target_dir)
497 {
498 find({
499 wanted => sub {
500 return if -d;
501 my $filename = $_;
502 substr($filename, 0, 1 + length $target_dir) = "";
503 return unless -f "$path/_static/$filename";
504 ln_sf($_, "$path/_static/$filename");
505 },
506 no_chdir => 1
507 }, $target_dir);
508
509 my $rtd_theme_version = get_installed_package_version("sphinx-rtd-theme-common");
510 $rtd_theme_version =~ s/-[^-]+$//; # Remove the Debian version suffix
511 push @deps, "sphinx-rtd-theme-common (>= $rtd_theme_version)";
512 }
513 return @deps;
514 }
515
516 sub get_installed_package_version($)
517 {
518 my ($package_name) = @_;
519 return `dpkg-query -W -f '\${Version}' $package_name 2>/dev/null`;
520 }
521
522 sub list_built_using($)
523 {
524 my ($path) = @_;
525 my @built_using;
526 my $sphinx_version = get_installed_package_version("sphinx-common");
527 push @built_using, "sphinx (= $sphinx_version)";
528 if (-d "$path/_static" and -f "$path/_static/alabaster.css")
529 {
530 my $alabaster_version = get_installed_package_version("python3-alabaster");
531 if ($alabaster_version)
532 {
533 push @built_using, "alabaster (= $alabaster_version)";
534 }
535 }
536 return @built_using;
537 }
538
539 sub fix_sphinx_doc($$)
540 {
541 my ($package, $path) = @_;
542 my $is_html = looks_like_sphinx_doc($path);
543 my $singlehtml_file = looks_like_sphinx_singlehtml_doc($path);
544 return 0 if not ($is_html or $singlehtml_file);
545 my @js = sanity_check($path, $singlehtml_file);
546 my @rtd_deps = process_rtd($path);
547 my @deps = fix_symlinks($path, @js);
548 my @built_using = list_built_using($path);
549 drop_cruft($path);
550 map { addsubstvar($package, "sphinxdoc:Depends", $_) } @deps;
551 map { addsubstvar($package, "sphinxdoc:Depends", $_) } @rtd_deps;
552 # Do not advertise sphinxdoc:Built-Using functionality to prevent its misuse,
553 # so add this variable only for packages that are already using it.
554 my $built_using_value = package_field($package, 'built-using', '');
555 if (index($built_using_value, '${sphinxdoc:Built-Using}') != -1)
556 {
557 map { addsubstvar($package, "sphinxdoc:Built-Using", $_) } @built_using;
558 }
559 return 1;
560 }
561
562 init();
563
564 load_packaged_js();
565
566 my @paths = @ARGV;
567 @paths = (undef) unless @paths;
568
569 foreach my $path (@paths)
570 {
571 my $done = 0;
572 my @matching_packages;
573 foreach my $package (@{$dh{DOPACKAGES}})
574 {
575 my $pkgpath = tmpdir($package);
576 if (defined $path)
577 {
578 next if -l $path;
579 $pkgpath .= "/$path";
580 next unless -d $pkgpath;
581 push @matching_packages, $package;
582 $done += fix_sphinx_doc($package, $pkgpath);
583 }
584 else
585 {
586 $pkgpath .= '/usr/share/doc/';
587 next unless -d $pkgpath;
588 find({
589 wanted => sub {
590 return unless -d;
591 return if -l;
592 return if excludefile($_);
593 $done += fix_sphinx_doc($package, $_);
594 },
595 no_chdir => 1
596 }, $pkgpath);
597 }
598 }
599 if ($done == 0)
600 {
601 if (defined $path)
602 {
603 if (!@matching_packages)
604 {
605 error("Path $path not found in any built package\n(searched in: @{$dh{DOPACKAGES}})");
606 }
607 error("Sphinx documentation not found at $path\n(path found in packages: @matching_packages)");
608 }
609 else
610 {
611 warning("Sphinx documentation not found");
612 }
613 }
614 }
615
616 =head1 SEE ALSO
617
618 L<debhelper(7)>, L<dh(1)>.
619
620 This program is meant to be used together with debhelper.
621
622 =head1 AUTHOR
623
624 Jakub Wilk <[email protected]>
625
626 =cut
627
628 # vim:ts=4 sw=4 et