Debian Codegraph @

src/main.c

raw
1 /* Command line parsing.
2 Copyright (C) 1996-2015, 2018-2024 Free Software Foundation, Inc.
3
4 This file is part of GNU Wget.
5
6 GNU Wget is free software; you can redistribute it and/or modify
7 it under the terms of the GNU General Public License as published by
8 the Free Software Foundation; either version 3 of the License, or
9 (at your option) any later version.
10
11 GNU Wget is distributed in the hope that it will be useful,
12 but WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 GNU General Public License for more details.
15
16 You should have received a copy of the GNU General Public License
17 along with Wget. If not, see <http://www.gnu.org/licenses/>.
18
19 Additional permission under GNU GPL version 3 section 7
20
21 If you modify this program, or any covered work, by linking or
22 combining it with the OpenSSL project's OpenSSL library (or a
23 modified version of that library), containing parts covered by the
24 terms of the OpenSSL or SSLeay licenses, the Free Software Foundation
25 grants you additional permission to convey the resulting work.
26 Corresponding Source for a non-source form of such a combination
27 shall include the source code for the parts of OpenSSL used as well
28 as that of the covered work. */
29
30 #include "wget.h"
31
32 #include <stdio.h>
33 #include <stdlib.h>
34 #include <unistd.h>
35 #include <string.h>
36 #include <signal.h>
37 #include <spawn.h>
38 #if defined(ENABLE_NLS) || defined(WINDOWS)
39 # include <locale.h>
40 #endif
41 #include <assert.h>
42 #include <errno.h>
43 #include <time.h>
44
45 #include "exits.h"
46 #include "utils.h"
47 #include "init.h"
48 #include "retr.h"
49 #include "recur.h"
50 #include "host.h"
51 #include "url.h"
52 #include "progress.h" /* for progress_handle_sigwinch */
53 #include "convert.h"
54 #include "spider.h"
55 #include "http.h" /* for save_cookies */
56 #include "hsts.h" /* for initializing hsts_store to NULL */
57 #include "ptimer.h"
58 #include "warc.h"
59 #include "version.h"
60 #include "c-strcase.h"
61 #include "dirname.h"
62 #include "xmemdup0.h"
63 #include <getopt.h>
64 #include <getpass.h>
65 #include <quote.h>
66
67 #ifdef TESTING
68 /* Rename the main function so we can have a main() in fuzzing code
69 and call the original main. */
70 # define main main_wget
71 #endif
72
73 #ifdef HAVE_METALINK
74 # include <metalink/metalink_parser.h>
75 # include "metalink.h"
76 #endif
77
78 #ifdef WINDOWS
79 # include <io.h>
80 # include <fcntl.h>
81 #ifndef ENABLE_NLS
82 # include <mbctype.h>
83 #endif
84 #endif
85
86 #ifdef __VMS
87 # include "vms.h"
88 #endif /* __VMS */
89
90 #ifndef PATH_SEPARATOR
91 # define PATH_SEPARATOR '/'
92 #endif
93
94 #ifndef ENABLE_IRI
95 struct iri dummy_iri;
96 #endif
97
98 #ifdef HAVE_LIBCARES
99 #include <ares.h>
100 ares_channel ares;
101 #else
102 void *ares;
103 #endif
104
105 struct options opt;
106
107 /* defined in version.c */
108 extern char *system_getrc;
109 /* Used for --version output in print_version */
110 #define MAX_CHARS_PER_LINE 72
111 #define TABULATION 4
112
113 const char *exec_name;
114
115 /* Number of successfully downloaded URLs */
116 int numurls = 0;
117
118 /* Initialize I18N/L10N. That amounts to invoking setlocale, and
119 setting up gettext's message catalog using bindtextdomain and
120 textdomain. Does nothing if NLS is disabled or missing. */
121
122 #if defined(SIGHUP) || defined(SIGUSR1)
123 /* Hangup signal handler. When wget receives SIGHUP or SIGUSR1, it
124 will proceed operation as usual, trying to write into a log file.
125 If that is impossible, the output will be turned off. */
126
127 static void
128 redirect_output_signal (int sig)
129 {
130 const char *signal_name = "WTF?!";
131
132 #ifdef SIGHUP
133 if (sig == SIGHUP)
134 signal_name = "SIGHUP";
135 #endif
136 #ifdef SIGUSR1
137 if (sig == SIGUSR1)
138 signal_name = "SIGUSR1";
139 #endif
140
141 redirect_output (true,signal_name);
142 progress_schedule_redirect ();
143 signal (sig, redirect_output_signal);
144 }
145 #endif /* defined(SIGHUP) || defined(SIGUSR1) */
146
147 static void
148 i18n_initialize (void)
149 {
150 /* ENABLE_NLS implies existence of functions invoked here. */
151 #ifdef ENABLE_NLS
152 /* Set the current locale. */
153 setlocale (LC_ALL, "");
154 /* Set the text message domain. */
155 bindtextdomain ("wget", LOCALEDIR);
156 bindtextdomain ("wget-gnulib", LOCALEDIR);
157 textdomain ("wget");
158 #elif defined WINDOWS
159 char MBCP[16] = "";
160 int CP;
161
162 CP = _getmbcp(); /* Consider it's different from default. */
163 if (CP > 0)
164 snprintf(MBCP, sizeof(MBCP), ".%d", CP);
165 setlocale(LC_ALL, MBCP);
166 #endif /* ENABLE_NLS */
167 }
168
169 #ifdef HAVE_HSTS
170 /* make the HSTS store global */
171 hsts_store_t hsts_store;
172
173 static char*
174 get_hsts_database (void)
175 {
176 if (opt.hsts_file)
177 return xstrdup (opt.hsts_file);
178
179 if (opt.homedir)
180 {
181 char *dir = ajoin_dir_file(opt.homedir, ".wget-hsts");
182 return dir;
183 }
184
185 return NULL;
186 }
187
188 static void
189 load_hsts (void)
190 {
191 if (!hsts_store)
192 {
193 char *filename = get_hsts_database ();
194
195 if (filename)
196 {
197 DEBUGP (("Reading HSTS entries from %s\n", filename));
198
199 hsts_store = hsts_store_open (filename);
200
201 if (!hsts_store)
202 logprintf (LOG_NOTQUIET, "ERROR: could not open HSTS store at '%s'. "
203 "HSTS will be disabled.\n",
204 filename);
205 }
206 else
207 logprintf (LOG_NOTQUIET, "ERROR: could not open HSTS store. HSTS will be disabled.\n");
208
209 xfree (filename);
210 }
211 }
212
213 static void
214 save_hsts (void)
215 {
216 if (hsts_store)
217 {
218 char *filename = get_hsts_database ();
219
220 if (filename && hsts_store_has_changed (hsts_store))
221 {
222 DEBUGP (("Saving HSTS entries to %s\n", filename));
223 hsts_store_save (hsts_store, filename);
224 }
225
226 hsts_store_close (hsts_store);
227 xfree (hsts_store);
228
229 xfree (filename);
230 }
231 }
232 #endif
233
234 /* Definition of command-line options. */
235
236 _Noreturn static void print_help (void);
237 _Noreturn static void print_version (void);
238
239 #ifdef HAVE_SSL
240 # define IF_SSL(a,b,c,d,e) { a, b, c, d , e },
241 #else
242 # define IF_SSL(a,b,c,d,e)
243 #endif
244
245 struct cmdline_option {
246 char long_name[MAX_LONGOPTION];
247 char short_name;
248 enum {
249 OPT_VALUE,
250 OPT_BOOLEAN,
251 OPT_FUNCALL,
252 /* Non-standard options that have to be handled specially in
253 main(). */
254 OPT__APPEND_OUTPUT,
255 OPT__CLOBBER,
256 OPT__DONT_REMOVE_LISTING,
257 OPT__EXECUTE,
258 OPT__NO,
259 OPT__PARENT
260 } type;
261 const void *data; /* for standard options */
262 int argtype; /* for non-standard options */
263 };
264
265 static struct cmdline_option option_data[] =
266 {
267 { "accept", 'A', OPT_VALUE, "accept", -1 },
268 { "accept-regex", 0, OPT_VALUE, "acceptregex", -1 },
269 { "adjust-extension", 'E', OPT_BOOLEAN, "adjustextension", -1 },
270 { "append-output", 'a', OPT__APPEND_OUTPUT, NULL, required_argument },
271 { "ask-password", 0, OPT_BOOLEAN, "askpassword", -1 },
272 { "auth-no-challenge", 0, OPT_BOOLEAN, "authnochallenge", -1 },
273 { "background", 'b', OPT_BOOLEAN, "background", -1 },
274 { "backup-converted", 'K', OPT_BOOLEAN, "backupconverted", -1 },
275 { "backups", 0, OPT_BOOLEAN, "backups", -1 },
276 { "base", 'B', OPT_VALUE, "base", -1 },
277 { "bind-address", 0, OPT_VALUE, "bindaddress", -1 },
278 #ifdef HAVE_LIBCARES
279 { "bind-dns-address", 0, OPT_VALUE, "binddnsaddress", -1 },
280 #endif
281 { "body-data", 0, OPT_VALUE, "bodydata", -1 },
282 { "body-file", 0, OPT_VALUE, "bodyfile", -1 },
283 IF_SSL ( "ca-certificate", 0, OPT_VALUE, "cacertificate", -1 )
284 IF_SSL ( "ca-directory", 0, OPT_VALUE, "cadirectory", -1 )
285 { "cache", 0, OPT_BOOLEAN, "cache", -1 },
286 IF_SSL ( "certificate", 0, OPT_VALUE, "certificate", -1 )
287 IF_SSL ( "certificate-type", 0, OPT_VALUE, "certificatetype", -1 )
288 IF_SSL ( "check-certificate", 0, OPT_BOOLEAN, "checkcertificate", -1 )
289 { "clobber", 0, OPT__CLOBBER, NULL, optional_argument },
290 #ifdef HAVE_LIBZ
291 { "compression", 0, OPT_VALUE, "compression", -1 },
292 #endif
293 { "config", 0, OPT_VALUE, "chooseconfig", -1 },
294 { "connect-timeout", 0, OPT_VALUE, "connecttimeout", -1 },
295 { "continue", 'c', OPT_BOOLEAN, "continue", -1 },
296 { "convert-file-only", 0, OPT_BOOLEAN, "convertfileonly", -1 },
297 { "convert-links", 'k', OPT_BOOLEAN, "convertlinks", -1 },
298 { "content-disposition", 0, OPT_BOOLEAN, "contentdisposition", -1 },
299 { "content-on-error", 0, OPT_BOOLEAN, "contentonerror", -1 },
300 { "cookies", 0, OPT_BOOLEAN, "cookies", -1 },
301 IF_SSL ( "crl-file", 0, OPT_VALUE, "crlfile", -1 )
302 { "cut-dirs", 0, OPT_VALUE, "cutdirs", -1 },
303 { "debug", 'd', OPT_BOOLEAN, "debug", -1 },
304 { "default-page", 0, OPT_VALUE, "defaultpage", -1 },
305 { "delete-after", 0, OPT_BOOLEAN, "deleteafter", -1 },
306 { "directories", 0, OPT_BOOLEAN, "dirstruct", -1 },
307 { "directory-prefix", 'P', OPT_VALUE, "dirprefix", -1 },
308 { "dns-cache", 0, OPT_BOOLEAN, "dnscache", -1 },
309 #ifdef HAVE_LIBCARES
310 { "dns-servers", 0, OPT_VALUE, "dnsservers", -1 },
311 #endif
312 { "dns-timeout", 0, OPT_VALUE, "dnstimeout", -1 },
313 { "domains", 'D', OPT_VALUE, "domains", -1 },
314 { "dont-remove-listing", 0, OPT__DONT_REMOVE_LISTING, NULL, no_argument },
315 { "dot-style", 0, OPT_VALUE, "dotstyle", -1 }, /* deprecated */
316 { "egd-file", 0, OPT_VALUE, "egdfile", -1 },
317 { "exclude-directories", 'X', OPT_VALUE, "excludedirectories", -1 },
318 { "exclude-domains", 0, OPT_VALUE, "excludedomains", -1 },
319 { "execute", 'e', OPT__EXECUTE, NULL, required_argument },
320 { "follow-ftp", 0, OPT_BOOLEAN, "followftp", -1 },
321 { "follow-tags", 0, OPT_VALUE, "followtags", -1 },
322 { "force-directories", 'x', OPT_BOOLEAN, "dirstruct", -1 },
323 { "force-html", 'F', OPT_BOOLEAN, "forcehtml", -1 },
324 { "ftp-password", 0, OPT_VALUE, "ftppassword", -1 },
325 #ifdef __VMS
326 { "ftp-stmlf", 0, OPT_BOOLEAN, "ftpstmlf", -1 },
327 #endif /* def __VMS */
328 { "ftp-user", 0, OPT_VALUE, "ftpuser", -1 },
329 IF_SSL ( "ftps-clear-data-connection", 0, OPT_BOOLEAN, "ftpscleardataconnection", -1 )
330 IF_SSL ( "ftps-fallback-to-ftp", 0, OPT_BOOLEAN, "ftpsfallbacktoftp", -1 )
331 IF_SSL ( "ftps-implicit", 0, OPT_BOOLEAN, "ftpsimplicit", -1 )
332 IF_SSL ( "ftps-resume-ssl", 0, OPT_BOOLEAN, "ftpsresumessl", -1 )
333 { "glob", 0, OPT_BOOLEAN, "glob", -1 },
334 { "header", 0, OPT_VALUE, "header", -1 },
335 { "help", 'h', OPT_FUNCALL, (void *)print_help, no_argument },
336 { "host-directories", 0, OPT_BOOLEAN, "addhostdir", -1 },
337 #ifdef HAVE_HSTS
338 { "hsts", 0, OPT_BOOLEAN, "hsts", -1},
339 { "hsts-file", 0, OPT_VALUE, "hstsfile", -1 },
340 #endif
341 { "html-extension", 'E', OPT_BOOLEAN, "adjustextension", -1 }, /* deprecated */
342 { "htmlify", 0, OPT_BOOLEAN, "htmlify", -1 },
343 { "http-keep-alive", 0, OPT_BOOLEAN, "httpkeepalive", -1 },
344 { "http-passwd", 0, OPT_VALUE, "httppassword", -1 }, /* deprecated */
345 { "http-password", 0, OPT_VALUE, "httppassword", -1 },
346 { "http-user", 0, OPT_VALUE, "httpuser", -1 },
347 IF_SSL ( "https-only", 0, OPT_BOOLEAN, "httpsonly", -1 )
348 { "ignore-case", 0, OPT_BOOLEAN, "ignorecase", -1 },
349 { "ignore-length", 0, OPT_BOOLEAN, "ignorelength", -1 },
350 { "ignore-tags", 0, OPT_VALUE, "ignoretags", -1 },
351 { "include-directories", 'I', OPT_VALUE, "includedirectories", -1 },
352 #ifdef ENABLE_IPV6
353 { "inet4-only", '4', OPT_BOOLEAN, "inet4only", -1 },
354 { "inet6-only", '6', OPT_BOOLEAN, "inet6only", -1 },
355 #endif
356 { "input-file", 'i', OPT_VALUE, "input", -1 },
357 #ifdef HAVE_METALINK
358 { "input-metalink", 0, OPT_VALUE, "inputmetalink", -1 },
359 #endif
360 { "iri", 0, OPT_BOOLEAN, "iri", -1 },
361 { "keep-badhash", 0, OPT_BOOLEAN, "keepbadhash", -1 },
362 { "keep-session-cookies", 0, OPT_BOOLEAN, "keepsessioncookies", -1 },
363 { "level", 'l', OPT_VALUE, "reclevel", -1 },
364 { "limit-rate", 0, OPT_VALUE, "limitrate", -1 },
365 { "load-cookies", 0, OPT_VALUE, "loadcookies", -1 },
366 { "local-encoding", 0, OPT_VALUE, "localencoding", -1 },
367 { "rejected-log", 0, OPT_VALUE, "rejectedlog", -1 },
368 { "max-redirect", 0, OPT_VALUE, "maxredirect", -1 },
369 #ifdef HAVE_METALINK
370 { "metalink-index", 0, OPT_VALUE, "metalinkindex", -1 },
371 { "metalink-over-http", 0, OPT_BOOLEAN, "metalinkoverhttp", -1 },
372 #endif
373 { "method", 0, OPT_VALUE, "method", -1 },
374 { "mirror", 'm', OPT_BOOLEAN, "mirror", -1 },
375 { "netrc", 0, OPT_BOOLEAN, "netrc", -1 },
376 { "no", 'n', OPT__NO, NULL, required_argument },
377 { "no-clobber", 0, OPT_BOOLEAN, "noclobber", -1 },
378 { "no-config", 0, OPT_BOOLEAN, "noconfig", -1},
379 { "no-parent", 0, OPT_BOOLEAN, "noparent", -1 },
380 { "output-document", 'O', OPT_VALUE, "outputdocument", -1 },
381 { "output-file", 'o', OPT_VALUE, "logfile", -1 },
382 { "page-requisites", 'p', OPT_BOOLEAN, "pagerequisites", -1 },
383 { "parent", 0, OPT__PARENT, NULL, optional_argument },
384 { "passive-ftp", 0, OPT_BOOLEAN, "passiveftp", -1 },
385 { "password", 0, OPT_VALUE, "password", -1 },
386 IF_SSL ( "pinnedpubkey", 0, OPT_VALUE, "pinnedpubkey", -1 )
387 { "post-data", 0, OPT_VALUE, "postdata", -1 },
388 { "post-file", 0, OPT_VALUE, "postfile", -1 },
389 { "prefer-family", 0, OPT_VALUE, "preferfamily", -1 },
390 #ifdef HAVE_METALINK
391 { "preferred-location", 0, OPT_VALUE, "preferredlocation", -1 },
392 #endif
393 { "preserve-permissions", 0, OPT_BOOLEAN, "preservepermissions", -1 },
394 IF_SSL ( "ciphers", 0, OPT_VALUE, "ciphers", -1 )
395 IF_SSL ( "private-key", 0, OPT_VALUE, "privatekey", -1 )
396 IF_SSL ( "private-key-type", 0, OPT_VALUE, "privatekeytype", -1 )
397 { "progress", 0, OPT_VALUE, "progress", -1 },
398 { "show-progress", 0, OPT_BOOLEAN, "showprogress", -1 },
399 { "protocol-directories", 0, OPT_BOOLEAN, "protocoldirectories", -1 },
400 { "proxy", 0, OPT_BOOLEAN, "useproxy", -1 },
401 { "proxy__compat", 'Y', OPT_VALUE, "useproxy", -1 }, /* back-compatible */
402 { "proxy-passwd", 0, OPT_VALUE, "proxypassword", -1 }, /* deprecated */
403 { "proxy-password", 0, OPT_VALUE, "proxypassword", -1 },
404 { "proxy-user", 0, OPT_VALUE, "proxyuser", -1 },
405 { "quiet", 'q', OPT_BOOLEAN, "quiet", -1 },
406 { "quota", 'Q', OPT_VALUE, "quota", -1 },
407 { "random-file", 0, OPT_VALUE, "randomfile", -1 },
408 { "random-wait", 0, OPT_BOOLEAN, "randomwait", -1 },
409 { "read-timeout", 0, OPT_VALUE, "readtimeout", -1 },
410 { "recursive", 'r', OPT_BOOLEAN, "recursive", -1 },
411 { "referer", 0, OPT_VALUE, "referer", -1 },
412 { "regex-type", 0, OPT_VALUE, "regextype", -1 },
413 { "reject", 'R', OPT_VALUE, "reject", -1 },
414 { "reject-regex", 0, OPT_VALUE, "rejectregex", -1 },
415 { "relative", 'L', OPT_BOOLEAN, "relativeonly", -1 },
416 { "remote-encoding", 0, OPT_VALUE, "remoteencoding", -1 },
417 { "remove-listing", 0, OPT_BOOLEAN, "removelisting", -1 },
418 { "report-speed", 0, OPT_BOOLEAN, "reportspeed", -1 },
419 { "restrict-file-names", 0, OPT_BOOLEAN, "restrictfilenames", -1 },
420 { "retr-symlinks", 0, OPT_BOOLEAN, "retrsymlinks", -1 },
421 { "retry-connrefused", 0, OPT_BOOLEAN, "retryconnrefused", -1 },
422 { "retry-on-host-error", 0, OPT_BOOLEAN, "retryonhosterror", -1 },
423 { "retry-on-http-error", 0, OPT_VALUE, "retryonhttperror", -1 },
424 { "save-cookies", 0, OPT_VALUE, "savecookies", -1 },
425 { "save-headers", 0, OPT_BOOLEAN, "saveheaders", -1 },
426 IF_SSL ( "secure-protocol", 0, OPT_VALUE, "secureprotocol", -1 )
427 { "server-response", 'S', OPT_BOOLEAN, "serverresponse", -1 },
428 { "span-hosts", 'H', OPT_BOOLEAN, "spanhosts", -1 },
429 { "spider", 0, OPT_BOOLEAN, "spider", -1 },
430 { "start-pos", 0, OPT_VALUE, "startpos", -1 },
431 { "strict-comments", 0, OPT_BOOLEAN, "strictcomments", -1 },
432 { "timeout", 'T', OPT_VALUE, "timeout", -1 },
433 { "timestamping", 'N', OPT_BOOLEAN, "timestamping", -1 },
434 { "if-modified-since", 0, OPT_BOOLEAN, "ifmodifiedsince", -1 },
435 { "tries", 't', OPT_VALUE, "tries", -1 },
436 { "unlink", 0, OPT_BOOLEAN, "unlink", -1 },
437 { "trust-server-names", 0, OPT_BOOLEAN, "trustservernames", -1 },
438 #ifndef __VMS
439 { "use-askpass", 0, OPT_VALUE, "useaskpass", -1},
440 #endif
441 { "use-server-timestamps", 0, OPT_BOOLEAN, "useservertimestamps", -1 },
442 { "user", 0, OPT_VALUE, "user", -1 },
443 { "user-agent", 'U', OPT_VALUE, "useragent", -1 },
444 { "verbose", 'v', OPT_BOOLEAN, "verbose", -1 },
445 { "version", 'V', OPT_FUNCALL, (void *) print_version, no_argument },
446 { "wait", 'w', OPT_VALUE, "wait", -1 },
447 { "waitretry", 0, OPT_VALUE, "waitretry", -1 },
448 { "warc-cdx", 0, OPT_BOOLEAN, "warccdx", -1 },
449 #ifdef HAVE_LIBZ
450 { "warc-compression", 0, OPT_BOOLEAN, "warccompression", -1 },
451 #endif
452 { "warc-dedup", 0, OPT_VALUE, "warccdxdedup", -1 },
453 { "warc-digests", 0, OPT_BOOLEAN, "warcdigests", -1 },
454 { "warc-file", 0, OPT_VALUE, "warcfile", -1 },
455 { "warc-header", 0, OPT_VALUE, "warcheader", -1 },
456 { "warc-keep-log", 0, OPT_BOOLEAN, "warckeeplog", -1 },
457 { "warc-max-size", 0, OPT_VALUE, "warcmaxsize", -1 },
458 { "warc-tempdir", 0, OPT_VALUE, "warctempdir", -1 },
459 #ifdef USE_WATT32
460 { "wdebug", 0, OPT_BOOLEAN, "wdebug", -1 },
461 #endif
462 #ifdef ENABLE_XATTR
463 { "xattr", 0, OPT_BOOLEAN, "xattr", -1 },
464 #endif
465 };
466
467 #undef IF_SSL
468
469 /* Return a string that contains S with "no-" prepended. The string
470 is NUL-terminated and allocated off static storage at Wget
471 startup. */
472
473 static char *
474 no_prefix (const char *s)
475 {
476 static char buffer[2048];
477 static char *p = buffer;
478
479 char *cp = p;
480 int size = 3 + strlen (s) + 1; /* "no-STRING\0" */
481 assert(p + size <= buffer + sizeof (buffer));
482
483 cp[0] = 'n', cp[1] = 'o', cp[2] = '-';
484 strcpy (cp + 3, s);
485 p += size;
486 return cp;
487 }
488
489 /* The arguments that that main passes to getopt_long. */
490 static struct option long_options[2 * countof (option_data) + 1];
491 static char short_options[128];
492
493 /* Mapping between short option chars and option_data indices. */
494 static unsigned char optmap[96];
495
496 /* Marker for `--no-FOO' values in long_options. */
497 #define BOOLEAN_NEG_MARKER 1024
498
499 /* Initialize the long_options array used by getopt_long from the data
500 in option_data. */
501
502 static void
503 init_switches (void)
504 {
505 static bool initialized;
506 char *p = short_options;
507 size_t i, o = 0;
508
509 if (initialized)
510 return;
511 initialized = 1;
512
513 for (i = 0; i < countof (option_data); i++)
514 {
515 struct cmdline_option *cmdopt = &option_data[i];
516 struct option *longopt;
517
518 longopt = &long_options[o++];
519 longopt->name = cmdopt->long_name;
520 longopt->val = i;
521 if (cmdopt->short_name)
522 {
523 *p++ = cmdopt->short_name;
524 optmap[cmdopt->short_name - 32] = longopt - long_options;
525 }
526 switch (cmdopt->type)
527 {
528 case OPT_VALUE:
529 longopt->has_arg = required_argument;
530 if (cmdopt->short_name)
531 *p++ = ':';
532 break;
533 case OPT_BOOLEAN:
534 /* Specify an optional argument for long options, so that
535 --option=off works the same as --no-option, for
536 compatibility with pre-1.10 Wget. However, don't specify
537 optional arguments short-option booleans because they
538 prevent combining of short options. */
539 longopt->has_arg = optional_argument;
540 /* For Boolean options, add the "--no-FOO" variant, which is
541 identical to "--foo", except it has opposite meaning and
542 it doesn't allow an argument. */
543 longopt = &long_options[o++];
544 longopt->name = no_prefix (cmdopt->long_name);
545 longopt->has_arg = no_argument;
546 /* Mask the value so we'll be able to recognize that we're
547 dealing with the false value. */
548 longopt->val = i | BOOLEAN_NEG_MARKER;
549 break;
550 default:
551 assert (cmdopt->argtype != -1);
552 longopt->has_arg = cmdopt->argtype;
553 if (cmdopt->short_name)
554 {
555 if (longopt->has_arg == required_argument)
556 *p++ = ':';
557 /* Don't handle optional_argument */
558 }
559 }
560 }
561 /* Terminate short_options. */
562 *p = '\0';
563 /* No need for xzero(long_options[o]) because its storage is static
564 and it will be zeroed by default. */
565 assert (o <= countof (long_options));
566 }
567
568 /* Print the usage message. */
569 static int
570 print_usage (_GL_UNUSED int error)
571 {
572 #ifndef TESTING
573 return fprintf (error ? stderr : stdout,
574 _("Usage: %s [OPTION]... [URL]...\n"), exec_name);
575 #else
576 return 0;
577 #endif
578 }
579
580 /* Print the help message, describing all the available options. If
581 you add an option, be sure to update this list. */
582 _Noreturn static void
583 print_help (void)
584 {
585 #ifndef TESTING
586 /* We split the help text this way to ease translation of individual
587 entries. */
588 static const char *help[] = {
589 "\n",
590 N_("\
591 Mandatory arguments to long options are mandatory for short options too.\n\n"),
592 N_("\
593 Startup:\n"),
594 N_("\
595 -V, --version display the version of Wget and exit\n"),
596 N_("\
597 -h, --help print this help\n"),
598 N_("\
599 -b, --background go to background after startup\n"),
600 N_("\
601 -e, --execute=COMMAND execute a `.wgetrc'-style command\n"),
602 "\n",
603
604 N_("\
605 Logging and input file:\n"),
606 N_("\
607 -o, --output-file=FILE log messages to FILE\n"),
608 N_("\
609 -a, --append-output=FILE append messages to FILE\n"),
610 #ifdef ENABLE_DEBUG
611 N_("\
612 -d, --debug print lots of debugging information\n"),
613 #endif
614 #ifdef USE_WATT32
615 N_("\
616 --wdebug print Watt-32 debug output\n"),
617 #endif
618 N_("\
619 -q, --quiet quiet (no output)\n"),
620 N_("\
621 -v, --verbose be verbose (this is the default)\n"),
622 N_("\
623 -nv, --no-verbose turn off verboseness, without being quiet\n"),
624 N_("\
625 --report-speed=TYPE output bandwidth as TYPE. TYPE can be bits\n"),
626 N_("\
627 -i, --input-file=FILE download URLs found in local or external FILE\n"),
628 #ifdef HAVE_METALINK
629 N_("\
630 --input-metalink=FILE download files covered in local Metalink FILE\n"),
631 #endif
632 N_("\
633 -F, --force-html treat input file as HTML\n"),
634 N_("\
635 -B, --base=URL resolves HTML input-file links (-i -F)\n\
636 relative to URL\n"),
637 N_("\
638 --config=FILE specify config file to use\n"),
639 N_("\
640 --no-config do not read any config file\n"),
641 N_("\
642 --rejected-log=FILE log reasons for URL rejection to FILE\n"),
643 "\n",
644
645 N_("\
646 Download:\n"),
647 N_("\
648 -t, --tries=NUMBER set number of retries to NUMBER (0 unlimits)\n"),
649 N_("\
650 --retry-connrefused retry even if connection is refused\n"),
651 N_("\
652 --retry-on-host-error consider host errors as non-fatal, transient errors\n"),
653 N_("\
654 --retry-on-http-error=ERRORS comma-separated list of HTTP errors to retry\n"),
655 N_("\
656 -O, --output-document=FILE write documents to FILE\n"),
657 N_("\
658 -nc, --no-clobber skip downloads that would download to\n\
659 existing files (overwriting them)\n"),
660 N_("\
661 --no-netrc don't try to obtain credentials from .netrc\n"),
662 N_("\
663 -c, --continue resume getting a partially-downloaded file\n"),
664 N_("\
665 --start-pos=OFFSET start downloading from zero-based position OFFSET\n"),
666 N_("\
667 --progress=TYPE select progress gauge type\n"),
668 N_("\
669 --show-progress display the progress bar in any verbosity mode\n"),
670 N_("\
671 -N, --timestamping don't re-retrieve files unless newer than\n\
672 local\n"),
673 N_("\
674 --no-if-modified-since don't use conditional if-modified-since get\n\
675 requests in timestamping mode\n"),
676 N_("\
677 --no-use-server-timestamps don't set the local file's timestamp by\n\
678 the one on the server\n"),
679 N_("\
680 -S, --server-response print server response\n"),
681 N_("\
682 --spider don't download anything\n"),
683 N_("\
684 -T, --timeout=SECONDS set all timeout values to SECONDS\n"),
685 #ifdef HAVE_LIBCARES
686 N_("\
687 --dns-servers=ADDRESSES list of DNS servers to query (comma separated)\n"),
688 N_("\
689 --bind-dns-address=ADDRESS bind DNS resolver to ADDRESS (hostname or IP) on local host\n"),
690 #endif
691 N_("\
692 --dns-timeout=SECS set the DNS lookup timeout to SECS\n"),
693 N_("\
694 --connect-timeout=SECS set the connect timeout to SECS\n"),
695 N_("\
696 --read-timeout=SECS set the read timeout to SECS\n"),
697 N_("\
698 -w, --wait=SECONDS wait SECONDS between retrievals\n\
699 (applies if more then 1 URL is to be retrieved)\n"),
700 N_("\
701 --waitretry=SECONDS wait 1..SECONDS between retries of a retrieval\n\
702 (applies if more then 1 URL is to be retrieved)\n"),
703 N_("\
704 --random-wait wait from 0.5*WAIT...1.5*WAIT secs between retrievals\n\
705 (applies if more then 1 URL is to be retrieved)\n"),
706 N_("\
707 --no-proxy explicitly turn off proxy\n"),
708 N_("\
709 -Q, --quota=NUMBER set retrieval quota to NUMBER\n"),
710 N_("\
711 --bind-address=ADDRESS bind to ADDRESS (hostname or IP) on local host\n"),
712 N_("\
713 --limit-rate=RATE limit download rate to RATE\n"),
714 N_("\
715 --no-dns-cache disable caching DNS lookups\n"),
716 N_("\
717 --restrict-file-names=OS restrict chars in file names to ones OS allows\n"),
718 N_("\
719 --ignore-case ignore case when matching files/directories\n"),
720 #ifdef ENABLE_IPV6
721 N_("\
722 -4, --inet4-only connect only to IPv4 addresses\n"),
723 N_("\
724 -6, --inet6-only connect only to IPv6 addresses\n"),
725 N_("\
726 --prefer-family=FAMILY connect first to addresses of specified family,\n\
727 one of IPv6, IPv4, or none\n"),
728 #endif
729 N_("\
730 --user=USER set both ftp and http user to USER\n"),
731 N_("\
732 --password=PASS set both ftp and http password to PASS\n"),
733 N_("\
734 --ask-password prompt for passwords\n"),
735 #ifndef __VMS
736 N_("\
737 --use-askpass=COMMAND specify credential handler for requesting \n\
738 username and password. If no COMMAND is \n\
739 specified the WGET_ASKPASS or the SSH_ASKPASS \n\
740 environment variable is used.\n"),
741 #endif
742 N_("\
743 --no-iri turn off IRI support\n"),
744 N_("\
745 --local-encoding=ENC use ENC as the local encoding for IRIs\n"),
746 N_("\
747 --remote-encoding=ENC use ENC as the default remote encoding\n"),
748 N_("\
749 --unlink remove file before clobber\n"),
750 #ifdef HAVE_METALINK
751 N_("\
752 --keep-badhash keep files with checksum mismatch (append .badhash)\n"),
753 N_("\
754 --metalink-index=NUMBER Metalink application/metalink4+xml metaurl ordinal NUMBER\n"),
755 N_("\
756 --metalink-over-http use Metalink metadata from HTTP response headers\n"),
757 N_("\
758 --preferred-location preferred location for Metalink resources\n"),
759 #endif
760 #ifdef ENABLE_XATTR
761 N_("\
762 --xattr turn on storage of metadata in extended file attributes\n"),
763 #endif
764 "\n",
765
766 N_("\
767 Directories:\n"),
768 N_("\
769 -nd, --no-directories don't create directories\n"),
770 N_("\
771 -x, --force-directories force creation of directories\n"),
772 N_("\
773 -nH, --no-host-directories don't create host directories\n"),
774 N_("\
775 --protocol-directories use protocol name in directories\n"),
776 N_("\
777 -P, --directory-prefix=PREFIX save files to PREFIX/..\n"),
778 N_("\
779 --cut-dirs=NUMBER ignore NUMBER remote directory components\n"),
780 "\n",
781
782 N_("\
783 HTTP options:\n"),
784 N_("\
785 --http-user=USER set http user to USER\n"),
786 N_("\
787 --http-password=PASS set http password to PASS\n"),
788 N_("\
789 --no-cache disallow server-cached data\n"),
790 N_ ("\
791 --default-page=NAME change the default page name (normally\n\
792 this is 'index.html'.)\n"),
793 N_("\
794 -E, --adjust-extension save HTML/CSS documents with proper extensions\n"),
795 N_("\
796 --ignore-length ignore 'Content-Length' header field\n"),
797 N_("\
798 --header=STRING insert STRING among the headers\n"),
799 #ifdef HAVE_LIBZ
800 N_("\
801 --compression=TYPE choose compression, one of auto, gzip and none. (default: none)\n"),
802 #endif
803 N_("\
804 --max-redirect maximum redirections allowed per page\n"),
805 N_("\
806 --proxy-user=USER set USER as proxy username\n"),
807 N_("\
808 --proxy-password=PASS set PASS as proxy password\n"),
809 N_("\
810 --referer=URL include 'Referer: URL' header in HTTP request\n"),
811 N_("\
812 --save-headers save the HTTP headers to file\n"),
813 N_("\
814 -U, --user-agent=AGENT identify as AGENT instead of Wget/VERSION\n"),
815 N_("\
816 --no-http-keep-alive disable HTTP keep-alive (persistent connections)\n"),
817 N_("\
818 --no-cookies don't use cookies\n"),
819 N_("\
820 --load-cookies=FILE load cookies from FILE before session\n"),
821 N_("\
822 --save-cookies=FILE save cookies to FILE after session\n"),
823 N_("\
824 --keep-session-cookies load and save session (non-permanent) cookies\n"),
825 N_("\
826 --post-data=STRING use the POST method; send STRING as the data\n"),
827 N_("\
828 --post-file=FILE use the POST method; send contents of FILE\n"),
829 N_("\
830 --method=HTTPMethod use method \"HTTPMethod\" in the request\n"),
831 N_("\
832 --body-data=STRING send STRING as data. --method MUST be set\n"),
833 N_("\
834 --body-file=FILE send contents of FILE. --method MUST be set\n"),
835 N_("\
836 --content-disposition honor the Content-Disposition header when\n\
837 choosing local file names (EXPERIMENTAL)\n"),
838 N_("\
839 --content-on-error output the received content on server errors\n"),
840 N_("\
841 --auth-no-challenge send Basic HTTP authentication information\n\
842 without first waiting for the server's\n\
843 challenge\n"),
844 "\n",
845
846 #ifdef HAVE_SSL
847 N_("\
848 HTTPS (SSL/TLS) options:\n"),
849 N_("\
850 --secure-protocol=PR choose secure protocol, one of auto, SSLv2,\n\
851 SSLv3, TLSv1, TLSv1_1, TLSv1_2, TLSv1_3 and PFS\n"),
852 N_("\
853 --https-only only follow secure HTTPS links\n"),
854 N_("\
855 --no-check-certificate don't validate the server's certificate\n"),
856 N_("\
857 --certificate=FILE client certificate file\n"),
858 N_("\
859 --certificate-type=TYPE client certificate type, PEM or DER\n"),
860 N_("\
861 --private-key=FILE private key file\n"),
862 N_("\
863 --private-key-type=TYPE private key type, PEM or DER\n"),
864 N_("\
865 --ca-certificate=FILE file with the bundle of CAs\n"),
866 N_("\
867 --ca-directory=DIR directory where hash list of CAs is stored\n"),
868 N_("\
869 --crl-file=FILE file with bundle of CRLs\n"),
870 N_("\
871 --pinnedpubkey=FILE/HASHES Public key (PEM/DER) file, or any number\n\
872 of base64 encoded sha256 hashes preceded by\n\
873 \'sha256//\' and separated by \';\', to verify\n\
874 peer against\n"),
875 #if defined(HAVE_LIBSSL) || defined(HAVE_LIBSSL32)
876 N_("\
877 --random-file=FILE file with random data for seeding the SSL PRNG\n"),
878 #endif
879 #if (defined(HAVE_LIBSSL) || defined(HAVE_LIBSSL32)) && defined(HAVE_RAND_EGD)
880 N_("\
881 --egd-file=FILE file naming the EGD socket with random data\n"),
882 #endif
883 "\n",
884 N_("\
885 --ciphers=STR Set the priority string (GnuTLS) or cipher list string (OpenSSL) directly.\n\
886 Use with care. This option overrides --secure-protocol.\n\
887 The format and syntax of this string depend on the specific SSL/TLS engine.\n"),
888 #endif /* HAVE_SSL */
889
890 #ifdef HAVE_HSTS
891 N_("\
892 HSTS options:\n"),
893 N_("\
894 --no-hsts disable HSTS\n"),
895 N_("\
896 --hsts-file path of HSTS database (will override default)\n"),
897 "\n",
898 #endif
899
900 N_("\
901 FTP options:\n"),
902 #ifdef __VMS
903 N_("\
904 --ftp-stmlf use Stream_LF format for all binary FTP files\n"),
905 #endif /* def __VMS */
906 N_("\
907 --ftp-user=USER set ftp user to USER\n"),
908 N_("\
909 --ftp-password=PASS set ftp password to PASS\n"),
910 N_("\
911 --no-remove-listing don't remove '.listing' files\n"),
912 N_("\
913 --no-glob turn off FTP file name globbing\n"),
914 N_("\
915 --no-passive-ftp disable the \"passive\" transfer mode\n"),
916 N_("\
917 --preserve-permissions preserve remote file permissions\n"),
918 N_("\
919 --retr-symlinks when recursing, get linked-to files (not dir)\n"),
920 "\n",
921
922 #ifdef HAVE_SSL
923 N_("\
924 FTPS options:\n"),
925 N_("\
926 --ftps-implicit use implicit FTPS (default port is 990)\n"),
927 N_("\
928 --ftps-resume-ssl resume the SSL/TLS session started in the control connection when\n"
929 " opening a data connection\n"),
930 N_("\
931 --ftps-clear-data-connection cipher the control channel only; all the data will be in plaintext\n"),
932 N_("\
933 --ftps-fallback-to-ftp fall back to FTP if FTPS is not supported in the target server\n"),
934 #endif
935
936 N_("\
937 WARC options:\n"),
938 N_("\
939 --warc-file=FILENAME save request/response data to a .warc.gz file\n"),
940 N_("\
941 --warc-header=STRING insert STRING into the warcinfo record\n"),
942 N_("\
943 --warc-max-size=NUMBER set maximum size of WARC files to NUMBER\n"),
944 N_("\
945 --warc-cdx write CDX index files\n"),
946 N_("\
947 --warc-dedup=FILENAME do not store records listed in this CDX file\n"),
948 #ifdef HAVE_LIBZ
949 N_("\
950 --no-warc-compression do not compress WARC files with GZIP\n"),
951 #endif
952 N_("\
953 --no-warc-digests do not calculate SHA1 digests\n"),
954 N_("\
955 --no-warc-keep-log do not store the log file in a WARC record\n"),
956 N_("\
957 --warc-tempdir=DIRECTORY location for temporary files created by the\n\
958 WARC writer\n"),
959 "\n",
960
961 N_("\
962 Recursive download:\n"),
963 N_("\
964 -r, --recursive specify recursive download\n"),
965 N_("\
966 -l, --level=NUMBER maximum recursion depth (inf or 0 for infinite)\n"),
967 N_("\
968 --delete-after delete files locally after downloading them\n"),
969 N_("\
970 -k, --convert-links make links in downloaded HTML or CSS point to\n\
971 local files\n"),
972 N_("\
973 --convert-file-only convert the file part of the URLs only (usually known as the basename)\n"),
974 N_("\
975 --backups=N before writing file X, rotate up to N backup files\n"),
976
977 #ifdef __VMS
978 N_("\
979 -K, --backup-converted before converting file X, back up as X_orig\n"),
980 #else /* def __VMS */
981 N_("\
982 -K, --backup-converted before converting file X, back up as X.orig\n"),
983 #endif /* def __VMS [else] */
984 N_("\
985 -m, --mirror shortcut for -N -r -l inf --no-remove-listing\n"),
986 N_("\
987 -p, --page-requisites get all images, etc. needed to display HTML page\n"),
988 N_("\
989 --strict-comments turn on strict (SGML) handling of HTML comments\n"),
990 "\n",
991
992 N_("\
993 Recursive accept/reject:\n"),
994 N_("\
995 -A, --accept=LIST comma-separated list of accepted extensions\n"),
996 N_("\
997 -R, --reject=LIST comma-separated list of rejected extensions\n"),
998 N_("\
999 --accept-regex=REGEX regex matching accepted URLs\n"),
1000 N_("\
1001 --reject-regex=REGEX regex matching rejected URLs\n"),
1002 #if defined HAVE_LIBPCRE || defined HAVE_LIBPCRE2
1003 N_("\
1004 --regex-type=TYPE regex type (posix|pcre)\n"),
1005 #else
1006 N_("\
1007 --regex-type=TYPE regex type (posix)\n"),
1008 #endif
1009 N_("\
1010 -D, --domains=LIST comma-separated list of accepted domains\n"),
1011 N_("\
1012 --exclude-domains=LIST comma-separated list of rejected domains\n"),
1013 N_("\
1014 --follow-ftp follow FTP links from HTML documents\n"),
1015 N_("\
1016 --follow-tags=LIST comma-separated list of followed HTML tags\n"),
1017 N_("\
1018 --ignore-tags=LIST comma-separated list of ignored HTML tags\n"),
1019 N_("\
1020 -H, --span-hosts go to foreign hosts when recursive\n"),
1021 N_("\
1022 -L, --relative follow relative links only\n"),
1023 N_("\
1024 -I, --include-directories=LIST list of allowed directories\n"),
1025 N_("\
1026 --trust-server-names use the name specified by the redirection\n\
1027 URL's last component\n"),
1028 N_("\
1029 -X, --exclude-directories=LIST list of excluded directories\n"),
1030 N_("\
1031 -np, --no-parent don't ascend to the parent directory\n"),
1032 "\n",
1033 N_("Email bug reports, questions, discussions to <[email protected]>\n"
1034 "and/or open issues at https://savannah.gnu.org/bugs/?func=additem&group=wget.\n")
1035 };
1036
1037 size_t i;
1038
1039 if (printf (_("GNU Wget %s, a non-interactive network retriever.\n"),
1040 version_string) < 0)
1041 exit (WGET_EXIT_IO_FAIL);
1042 if (print_usage (0) < 0)
1043 exit (WGET_EXIT_IO_FAIL);
1044
1045 for (i = 0; i < countof (help); i++)
1046 if (fputs (_(help[i]), stdout) < 0)
1047 exit (WGET_EXIT_IO_FAIL);
1048 #endif /* TESTING */
1049 exit (WGET_EXIT_SUCCESS);
1050 }
1051
1052 /* Return a human-readable printed representation of INTERVAL,
1053 measured in seconds. */
1054
1055 static char *
1056 secs_to_human_time (double interval)
1057 {
1058 static char buf[32];
1059 int secs = (int) (interval + 0.5);
1060 int hours, mins, days;
1061
1062 days = secs / 86400, secs %= 86400;
1063 hours = secs / 3600, secs %= 3600;
1064 mins = secs / 60, secs %= 60;
1065
1066 if (days)
1067 snprintf (buf, sizeof(buf), "%dd %dh %dm %ds", days, hours, mins, secs);
1068 else if (hours)
1069 snprintf (buf, sizeof(buf), "%dh %dm %ds", hours, mins, secs);
1070 else if (mins)
1071 snprintf (buf, sizeof(buf), "%dm %ds", mins, secs);
1072 else
1073 snprintf (buf, sizeof(buf), "%ss", print_decimal (interval));
1074
1075 return buf;
1076 }
1077
1078 static char *
1079 prompt_for_password (void)
1080 {
1081 if (opt.user)
1082 fprintf (stderr, _("Password for user %s: "), quote (opt.user));
1083 else
1084 fprintf (stderr, _("Password: "));
1085 #ifndef TESTING
1086 /* gnulib's getpass() uses static variables internally, bad for fuzing */
1087 return getpass("");
1088 #else
1089 return xstrdup("");
1090 #endif
1091 }
1092
1093
1094 /* Execute external application opt.use_askpass */
1095 static void
1096 run_use_askpass (char *question, char **answer)
1097 {
1098 char tmp[1024];
1099 pid_t pid;
1100 int status;
1101 int com[2];
1102 ssize_t bytes = 0;
1103 char *argv[3], *p;
1104 posix_spawn_file_actions_t fa;
1105
1106 if (pipe (com) == -1)
1107 {
1108 fprintf (stderr, _("Cannot create pipe\n"));
1109 exit (WGET_EXIT_GENERIC_ERROR);
1110 }
1111
1112 status = posix_spawn_file_actions_init (&fa);
1113 if (status)
1114 {
1115 fprintf (stderr,
1116 _("Error initializing spawn file actions for use-askpass: %d\n"),
1117 status);
1118 exit (WGET_EXIT_GENERIC_ERROR);
1119 }
1120
1121 status = posix_spawn_file_actions_adddup2 (&fa, com[1], STDOUT_FILENO);
1122 if (status)
1123 {
1124 fprintf (stderr,
1125 _("Error setting spawn file actions for use-askpass: %d\n"),
1126 status);
1127 exit (WGET_EXIT_GENERIC_ERROR);
1128 }
1129
1130 /* C89 initializer lists must be computable at load time,
1131 * thus this explicit initialization. */
1132 argv[0] = opt.use_askpass;
1133 argv[1] = question;
1134 argv[2] = NULL;
1135
1136 status = posix_spawnp (&pid, opt.use_askpass, &fa, NULL, argv, environ);
1137 if (status)
1138 {
1139 fprintf (stderr, "Error spawning %s: %d\n", opt.use_askpass, status);
1140 exit (WGET_EXIT_GENERIC_ERROR);
1141 }
1142
1143 /* Parent process reads from child. */
1144 close (com[1]);
1145 bytes = read (com[0], tmp, sizeof (tmp) - 1);
1146 if (bytes <= 0)
1147 {
1148 fprintf (stderr,
1149 _("Error reading response from command \"%s %s\": %s\n"),
1150 opt.use_askpass, question, strerror (errno));
1151 exit (WGET_EXIT_GENERIC_ERROR);
1152 }
1153
1154 /* Make sure there is a trailing 0 */
1155 tmp[bytes] = '\0';
1156
1157 /* Remove a possible new line */
1158 if ((p = strpbrk (tmp, "\r\n")))
1159 bytes = p - tmp;
1160
1161 *answer = xmemdup0 (tmp, bytes);
1162 }
1163
1164 /* set the user name and password*/
1165 static void
1166 use_askpass (struct url *u)
1167 {
1168 static char question[1024];
1169
1170 if (u->user == NULL || u->user[0] == '\0')
1171 {
1172 snprintf (question, sizeof (question), _("Username for '%s%s': "),
1173 scheme_leading_string(u->scheme), u->host);
1174 /* Prompt for username */
1175 run_use_askpass (question, &u->user);
1176 if (opt.recursive)
1177 opt.user = xstrdup (u->user);
1178 }
1179
1180 if (u->passwd == NULL || u->passwd[0] == '\0')
1181 {
1182 snprintf(question, sizeof (question), _("Password for '%s%s@%s': "),
1183 scheme_leading_string (u->scheme), u->user, u->host);
1184 /* Prompt for password */
1185 run_use_askpass (question, &u->passwd);
1186 if (opt.recursive)
1187 opt.passwd = xstrdup (u->passwd);
1188 }
1189 }
1190 /* Function that prints the line argument while limiting it
1191 to at most line_length. prefix is printed on the first line
1192 and an appropriate number of spaces are added on subsequent
1193 lines.*/
1194 static int
1195 format_and_print_line (const char *prefix, const char *line,
1196 int line_length)
1197 {
1198 int remaining_chars;
1199 char *line_dup, *token;
1200
1201 assert (prefix != NULL);
1202 assert (line != NULL);
1203 assert (line_length > TABULATION);
1204
1205 line_dup = xstrdup (line);
1206
1207 if (printf ("%s", prefix) < 0)
1208 {
1209 xfree (line_dup);
1210 return -1;
1211 }
1212
1213 /* Wrap to new line after prefix. */
1214 remaining_chars = 0;
1215
1216 /* We break on spaces. */
1217 token = strtok (line_dup, " ");
1218 while (token != NULL)
1219 {
1220 /* If however a token is much larger than the maximum
1221 line length, all bets are off and we simply print the
1222 token on the next line. */
1223 if (remaining_chars <= (int) strlen (token))
1224 {
1225 if (printf ("\n%*c", TABULATION, ' ') < 0)
1226 {
1227 xfree (line_dup);
1228 return -1;
1229 }
1230 remaining_chars = line_length - TABULATION;
1231 }
1232 if (printf ("%s ", token) < 0)
1233 {
1234 xfree (line_dup);
1235 return -1;
1236 }
1237 remaining_chars -= strlen (token) + 1; /* account for " " */
1238 token = strtok (NULL, " ");
1239 }
1240
1241 if (printf ("\n") < 0)
1242 {
1243 xfree (line_dup);
1244 return -1;
1245 }
1246
1247 xfree (line_dup);
1248 return 0;
1249 }
1250
1251 _Noreturn static void
1252 print_version (void)
1253 {
1254 const char *wgetrc_title = _("Wgetrc: ");
1255 const char *locale_title = _("Locale: ");
1256 const char *compile_title = _("Compile: ");
1257 const char *link_title = _("Link: ");
1258 char *env_wgetrc, *user_wgetrc;
1259 int i;
1260
1261 if (printf (_("GNU Wget %s built on %s.\n\n"), version_string, OS_TYPE) < 0)
1262 exit (WGET_EXIT_IO_FAIL);
1263
1264 for (i = 0; compiled_features[i] != NULL; )
1265 {
1266 int line_length = MAX_CHARS_PER_LINE;
1267 while ((line_length > 0) && (compiled_features[i] != NULL))
1268 {
1269 if (printf ("%s ", compiled_features[i]) < 0)
1270 exit (WGET_EXIT_IO_FAIL);
1271 line_length -= (int) strlen (compiled_features[i]) + 2;
1272 i++;
1273 }
1274 if (printf ("\n") < 0)
1275 exit (WGET_EXIT_IO_FAIL);
1276 }
1277 if (printf ("\n") < 0)
1278 exit (WGET_EXIT_IO_FAIL);
1279
1280 /* Print VMS-specific version info. */
1281 #ifdef __VMS
1282 if (vms_version_supplement() < 0)
1283 exit (WGET_EXIT_IO_FAIL);
1284 #endif /* def __VMS */
1285
1286 /* Handle the case when $WGETRC is unset and $HOME/.wgetrc is
1287 absent. */
1288 if (printf ("%s\n", wgetrc_title) < 0)
1289 exit (WGET_EXIT_IO_FAIL);
1290
1291 env_wgetrc = wgetrc_env_file_name ();
1292 if (env_wgetrc && *env_wgetrc)
1293 {
1294 if (printf (_(" %s (env)\n"), env_wgetrc) < 0)
1295 exit (WGET_EXIT_IO_FAIL);
1296 xfree (env_wgetrc);
1297 }
1298 user_wgetrc = wgetrc_user_file_name ();
1299 if (user_wgetrc)
1300 {
1301 if (printf (_(" %s (user)\n"), user_wgetrc) < 0)
1302 exit (WGET_EXIT_IO_FAIL);
1303 xfree (user_wgetrc);
1304 }
1305 #ifdef SYSTEM_WGETRC
1306 if (printf (_(" %s (system)\n"), SYSTEM_WGETRC) < 0)
1307 exit (WGET_EXIT_IO_FAIL);
1308 #endif
1309
1310 #ifdef ENABLE_NLS
1311 if (format_and_print_line (locale_title,
1312 LOCALEDIR,
1313 MAX_CHARS_PER_LINE) < 0)
1314 exit (WGET_EXIT_IO_FAIL);
1315 #endif /* def ENABLE_NLS */
1316
1317 if (compilation_string != NULL)
1318 if (format_and_print_line (compile_title,
1319 compilation_string,
1320 MAX_CHARS_PER_LINE) < 0)
1321 exit (WGET_EXIT_IO_FAIL);
1322
1323 if (link_string != NULL)
1324 if (format_and_print_line (link_title,
1325 link_string,
1326 MAX_CHARS_PER_LINE) < 0)
1327 exit (WGET_EXIT_IO_FAIL);
1328
1329 if (printf ("\n") < 0)
1330 exit (WGET_EXIT_IO_FAIL);
1331
1332 /* TRANSLATORS: When available, an actual copyright character
1333 (circle-c) should be used in preference to "(C)". */
1334 if (printf (_("\
1335 Copyright (C) %s Free Software Foundation, Inc.\n"), "2015") < 0)
1336 exit (WGET_EXIT_IO_FAIL);
1337 if (fputs (_("\
1338 License GPLv3+: GNU GPL version 3 or later\n\
1339 <http://www.gnu.org/licenses/gpl.html>.\n\
1340 This is free software: you are free to change and redistribute it.\n\
1341 There is NO WARRANTY, to the extent permitted by law.\n"), stdout) < 0)
1342 exit (WGET_EXIT_IO_FAIL);
1343 /* TRANSLATORS: When available, please use the proper diacritics for
1344 names such as this one. See en_US.po for reference. */
1345 if (fputs (_("\nOriginally written by Hrvoje Niksic <[email protected]>.\n"),
1346 stdout) < 0)
1347 exit (WGET_EXIT_IO_FAIL);
1348 if (fputs (_("Please send bug reports and questions to <[email protected]>.\n"),
1349 stdout) < 0)
1350 exit (WGET_EXIT_IO_FAIL);
1351
1352 exit (WGET_EXIT_SUCCESS);
1353 }
1354
1355 const char *program_name; /* Needed by lib/error.c. */
1356 const char *program_argstring; /* Needed by wget_warc.c. */
1357 struct ptimer *timer;
1358 int cleaned_up;
1359
1360 int
1361 main (int argc, char **argv)
1362 {
1363 char *p;
1364 int i, ret, longindex;
1365 int nurls;
1366 int argstring_length;
1367 bool use_userconfig = false;
1368 bool noconfig = false;
1369 bool append_to_log = false;
1370
1371 cleaned_up = 0; /* do cleanup later */
1372
1373 timer = ptimer_new ();
1374 double start_time = ptimer_measure (timer);
1375
1376 total_downloaded_bytes = 0;
1377 program_name = argv[0];
1378
1379 i18n_initialize ();
1380
1381 /* Construct the name of the executable, without the directory part. */
1382 #ifdef __VMS
1383 /* On VMS, lose the "dev:[dir]" prefix and the ".EXE;nnn" suffix. */
1384 exec_name = vms_basename (argv[0]);
1385 #else /* def __VMS */
1386 exec_name = base_name (argv[0]);
1387 #endif /* def __VMS [else] */
1388
1389 #ifdef WINDOWS
1390 /* Drop extension (typically .EXE) from executable filename. */
1391 windows_main ((char **) &exec_name);
1392 #endif
1393
1394 /* Construct the arguments string. */
1395 for (argstring_length = 1, i = 1; i < argc; i++)
1396 argstring_length += strlen (argv[i]) + 3 + 1;
1397 program_argstring = p = malloc (argstring_length);
1398 if (p == NULL)
1399 {
1400 fprintf (stderr, _("Memory allocation problem\n"));
1401 exit (WGET_EXIT_PARSE_ERROR);
1402 }
1403 for (i = 1; i < argc; i++)
1404 {
1405 int arglen;
1406
1407 *p++ = '"';
1408 arglen = strlen (argv[i]);
1409 memcpy (p, argv[i], arglen);
1410 p += arglen;
1411 *p++ = '"';
1412 *p++ = ' ';
1413 }
1414 *p = '\0';
1415
1416 /* Load the hard-coded defaults. */
1417 defaults ();
1418 opt.homedir = home_dir();
1419
1420 init_switches ();
1421
1422 /* This separate getopt_long is needed to find the user config file
1423 option ("--config") and parse it before the other user options. */
1424 longindex = -1;
1425
1426 while ((getopt_long (argc, argv,
1427 short_options, long_options, &longindex)) != -1)
1428 {
1429 int confval;
1430 struct cmdline_option *config_opt;
1431
1432 /* There is no short option for "--config". */
1433 if (longindex >= 0)
1434 {
1435 confval = long_options[longindex].val;
1436 config_opt = &option_data[confval & ~BOOLEAN_NEG_MARKER];
1437 if (strcmp (config_opt->long_name, "no-config") == 0)
1438 {
1439 noconfig = true;
1440 break;
1441 }
1442 else if (strcmp (config_opt->long_name, "config") == 0)
1443 {
1444 file_stats_t flstats;
1445 use_userconfig = true;
1446 memset(&flstats, 0, sizeof(flstats));
1447 if (file_exists_p(optarg, &flstats) && run_wgetrc (optarg, &flstats))
1448 break;
1449 else
1450 {
1451 fprintf (stderr, _("Exiting due to error in %s\n"), optarg);
1452 exit (WGET_EXIT_PARSE_ERROR);
1453 }
1454 }
1455 }
1456 }
1457
1458 /* If the user did not specify a config, read the system wgetrc and ~/.wgetrc. */
1459 if (noconfig == false && use_userconfig == false)
1460 if ((ret = initialize ()))
1461 return ret;
1462
1463 opterr = 0;
1464 optind = 0;
1465
1466 longindex = -1;
1467 while ((ret = getopt_long (argc, argv,
1468 short_options, long_options, &longindex)) != -1)
1469 {
1470 int val;
1471 struct cmdline_option *cmdopt;
1472
1473 /* If LONGINDEX is unchanged, it means RET is referring a short
1474 option. */
1475 if (longindex == -1)
1476 {
1477 if (ret == '?')
1478 {
1479 print_usage (1);
1480 fprintf (stderr, "\n");
1481 fprintf (stderr, _("Try `%s --help' for more options.\n"),
1482 exec_name);
1483 exit (WGET_EXIT_PARSE_ERROR);
1484 }
1485 /* Find the short option character in the mapping. */
1486 longindex = optmap[ret - 32];
1487 }
1488 val = long_options[longindex].val;
1489
1490 /* Use the retrieved value to locate the option in the
1491 option_data array, and to see if we're dealing with the
1492 negated "--no-FOO" variant of the boolean option "--foo". */
1493 cmdopt = &option_data[val & ~BOOLEAN_NEG_MARKER];
1494 switch (cmdopt->type)
1495 {
1496 case OPT_VALUE:
1497 setoptval (cmdopt->data, optarg, cmdopt->long_name);
1498 break;
1499 case OPT_BOOLEAN:
1500 if (optarg)
1501 /* The user has specified a value -- use it. */
1502 setoptval (cmdopt->data, optarg, cmdopt->long_name);
1503 else
1504 {
1505 /* NEG is true for `--no-FOO' style boolean options. */
1506 bool neg = !!(val & BOOLEAN_NEG_MARKER);
1507 setoptval (cmdopt->data, neg ? "0" : "1", cmdopt->long_name);
1508 }
1509 break;
1510 case OPT_FUNCALL:
1511 {
1512 void (*func) (void) = (void (*) (void)) cmdopt->data;
1513 func ();
1514 }
1515 break;
1516 case OPT__APPEND_OUTPUT:
1517 setoptval ("logfile", optarg, cmdopt->long_name);
1518 append_to_log = true;
1519 break;
1520 case OPT__EXECUTE:
1521 if (optarg) /* check silences static analyzer */
1522 run_command (optarg);
1523 break;
1524 case OPT__NO:
1525 {
1526 /* We support real --no-FOO flags now, but keep these
1527 short options for convenience and backward
1528 compatibility. */
1529 for (p = optarg; p && *p; p++)
1530 switch (*p)
1531 {
1532 case 'v':
1533 setoptval ("verbose", "0", cmdopt->long_name);
1534 break;
1535 case 'H':
1536 setoptval ("addhostdir", "0", cmdopt->long_name);
1537 break;
1538 case 'd':
1539 setoptval ("dirstruct", "0", cmdopt->long_name);
1540 break;
1541 case 'c':
1542 setoptval ("noclobber", "1", cmdopt->long_name);
1543 break;
1544 case 'p':
1545 setoptval ("noparent", "1", cmdopt->long_name);
1546 break;
1547 default:
1548 fprintf (stderr, _("%s: illegal option -- `-n%c'\n"),
1549 exec_name, *p);
1550 print_usage (1);
1551 fprintf (stderr, "\n");
1552 fprintf (stderr, _("Try `%s --help' for more options.\n"),
1553 exec_name);
1554 exit (WGET_EXIT_GENERIC_ERROR);
1555 }
1556 break;
1557 }
1558 case OPT__PARENT:
1559 case OPT__CLOBBER:
1560 {
1561 /* The wgetrc commands are named noparent and noclobber,
1562 so we must revert the meaning of the cmdline options
1563 before passing the value to setoptval. */
1564 bool flag = true;
1565 if (optarg)
1566 flag = (*optarg == '1' || c_tolower (*optarg) == 'y'
1567 || (c_tolower (optarg[0]) == 'o'
1568 && c_tolower (optarg[1]) == 'n'));
1569 setoptval (cmdopt->type == OPT__PARENT ? "noparent" : "noclobber",
1570 flag ? "0" : "1", cmdopt->long_name);
1571 break;
1572 }
1573 case OPT__DONT_REMOVE_LISTING:
1574 setoptval ("removelisting", "0", cmdopt->long_name);
1575 break;
1576 }
1577
1578 longindex = -1;
1579 }
1580
1581 nurls = argc - optind;
1582
1583 /* Initialize logging ASAP. */
1584 log_init (opt.lfilename, append_to_log);
1585
1586 /* If we do not have Debug support compiled in AND Wget is invoked with the
1587 * --debug switch, instead of failing, we silently turn it into a no-op. For
1588 * this no-op, we explicitly set opt.debug to false and hence none of the
1589 * Debug output messages will be printed.
1590 */
1591 #ifndef ENABLE_DEBUG
1592 if (opt.debug)
1593 {
1594 fprintf (stderr, _("Debugging support not compiled in. "
1595 "Ignoring --debug flag.\n"));
1596 opt.debug = false;
1597 }
1598 #endif
1599
1600 /* All user options have now been processed, so it's now safe to do
1601 interoption dependency checks. */
1602
1603 if (opt.noclobber && (opt.convert_links || opt.convert_file_only))
1604 {
1605 fprintf (stderr,
1606 opt.convert_links ?
1607 _("Both --no-clobber and --convert-links were specified,"
1608 " only --convert-links will be used.\n") :
1609 _("Both --no-clobber and --convert-file-only were specified,"
1610 " only --convert-file-only will be used.\n"));
1611 opt.noclobber = false;
1612 }
1613
1614 if (opt.reclevel == 0)
1615 opt.reclevel = INFINITE_RECURSION; /* see recur.h for commentary */
1616
1617 if (opt.spider || opt.delete_after)
1618 opt.no_dirstruct = true;
1619
1620 if (opt.page_requisites && !opt.recursive)
1621 {
1622 /* Don't set opt.recursive here because it would confuse the FTP
1623 code. Instead, call retrieve_tree below when either
1624 page_requisites or recursive is requested. */
1625 opt.reclevel = 0;
1626 if (!opt.no_dirstruct)
1627 opt.dirstruct = 1; /* normally handled by cmd_spec_recursive() */
1628 }
1629
1630 if (opt.verbose == -1)
1631 opt.verbose = !opt.quiet;
1632
1633 if (!opt.verbose && opt.show_progress == -1)
1634 opt.show_progress = false;
1635
1636 if (opt.quiet && opt.show_progress == -1)
1637 opt.show_progress = false;
1638
1639 /* Sanity checks. */
1640 if (opt.verbose && opt.quiet)
1641 {
1642 fprintf (stderr, _("Can't be verbose and quiet at the same time.\n"));
1643 print_usage (1);
1644 exit (WGET_EXIT_GENERIC_ERROR);
1645 }
1646 if (opt.timestamping && opt.noclobber)
1647 {
1648 fprintf (stderr, _("\
1649 Can't timestamp and not clobber old files at the same time.\n"));
1650 print_usage (1);
1651 exit (WGET_EXIT_GENERIC_ERROR);
1652 }
1653 #ifdef ENABLE_IPV6
1654 if (opt.ipv4_only && opt.ipv6_only)
1655 {
1656 fprintf (stderr,
1657 _("Cannot specify both --inet4-only and --inet6-only.\n"));
1658 print_usage (1);
1659 exit (WGET_EXIT_GENERIC_ERROR);
1660 }
1661 #endif
1662 if (opt.output_document)
1663 {
1664 if ((opt.convert_links || opt.convert_file_only)
1665 && (nurls > 1 || opt.page_requisites || opt.recursive))
1666 {
1667 fputs (_("\
1668 Cannot specify both -k or --convert-file-only and -O if multiple URLs are given, or in combination\n\
1669 with -p or -r. See the manual for details.\n\n"), stderr);
1670 print_usage (1);
1671 exit (WGET_EXIT_GENERIC_ERROR);
1672 }
1673 if (opt.page_requisites
1674 || opt.recursive)
1675 {
1676 logprintf (LOG_NOTQUIET, "%s", _("\
1677 WARNING: combining -O with -r or -p will mean that all downloaded content\n\
1678 will be placed in the single file you specified.\n\n"));
1679 }
1680 if (opt.timestamping)
1681 {
1682 logprintf (LOG_NOTQUIET, "%s", _("\
1683 WARNING: timestamping does nothing in combination with -O. See the manual\n\
1684 for details.\n\n"));
1685 opt.timestamping = false;
1686 }
1687 if (opt.noclobber && file_exists_p(opt.output_document, NULL))
1688 {
1689 /* Check if output file exists; if it does, exit. */
1690 logprintf (LOG_VERBOSE,
1691 _("File %s already there; not retrieving.\n"),
1692 quote (opt.output_document));
1693 exit (WGET_EXIT_GENERIC_ERROR);
1694 }
1695 }
1696
1697 if (opt.warc_filename != 0)
1698 {
1699 if (opt.noclobber)
1700 {
1701 fprintf (stderr,
1702 _("WARC output does not work with --no-clobber, "
1703 "--no-clobber will be disabled.\n"));
1704 opt.noclobber = false;
1705 }
1706 if (opt.timestamping)
1707 {
1708 fprintf (stderr,
1709 _("WARC output does not work with timestamping, "
1710 "timestamping will be disabled.\n"));
1711 opt.timestamping = false;
1712 }
1713 if (opt.spider)
1714 {
1715 fprintf (stderr,
1716 _("WARC output does not work with --spider.\n"));
1717 exit (WGET_EXIT_GENERIC_ERROR);
1718 }
1719 if (opt.always_rest || opt.start_pos >= 0)
1720 {
1721 fprintf (stderr,
1722 _("WARC output does not work with --continue or"
1723 " --start-pos, they will be disabled.\n"));
1724 opt.always_rest = false;
1725 opt.start_pos = -1;
1726 }
1727 if (opt.warc_cdx_dedup_filename != 0 && !opt.warc_digests_enabled)
1728 {
1729 fprintf (stderr,
1730 _("Digests are disabled; WARC deduplication will "
1731 "not find duplicate records.\n"));
1732 }
1733 if (opt.warc_keep_log)
1734 {
1735 opt.progress_type = xstrdup ("dot");
1736 }
1737 }
1738
1739 #ifdef HAVE_LIBZ
1740 if (opt.always_rest || opt.start_pos >= 0)
1741 {
1742 if (opt.compression == compression_auto)
1743 {
1744 /* Compression does not work with --continue or --start-pos.
1745 Since compression was not explicitly set, it will be disabled. */
1746 opt.compression = compression_none;
1747 }
1748 else if (opt.compression != compression_none)
1749 {
1750 fprintf (stderr,
1751 _("Compression does not work with --continue or"
1752 " --start-pos, they will be disabled.\n"));
1753 opt.always_rest = false;
1754 opt.start_pos = -1;
1755 }
1756 }
1757 #endif
1758
1759 if (opt.ask_passwd && opt.passwd)
1760 {
1761 fprintf (stderr,
1762 _("Cannot specify both --ask-password and --password.\n"));
1763 print_usage (1);
1764 exit (WGET_EXIT_GENERIC_ERROR);
1765 }
1766
1767 if (opt.ask_passwd && !(opt.user || opt.http_user || opt.ftp_user))
1768 {
1769 fprintf(stderr,
1770 _("WARNING: No username set with --ask-password. This is usually not what you want.\n"));
1771 }
1772
1773 if (opt.start_pos >= 0 && opt.always_rest)
1774 {
1775 fprintf (stderr,
1776 _("Specifying both --start-pos and --continue is not "
1777 "recommended; --continue will be disabled.\n"));
1778 opt.always_rest = false;
1779 }
1780
1781 if (!nurls && !opt.input_filename
1782 #ifdef HAVE_METALINK
1783 && !opt.input_metalink
1784 #endif
1785 )
1786 {
1787 /* No URL specified. */
1788 #ifndef TESTING
1789 fprintf (stderr, _("%s: missing URL\n"), exec_name);
1790 print_usage (1);
1791 fprintf (stderr, "\n");
1792 /* #### Something nicer should be printed here -- similar to the
1793 pre-1.5 `--help' page. */
1794 fprintf (stderr, _("Try `%s --help' for more options.\n"), exec_name);
1795 #endif
1796 exit (WGET_EXIT_GENERIC_ERROR);
1797 }
1798
1799 /* Compile the regular expressions. */
1800 switch (opt.regex_type)
1801 {
1802 #ifdef HAVE_LIBPCRE2
1803 case regex_type_pcre:
1804 opt.regex_compile_fun = compile_pcre2_regex;
1805 opt.regex_match_fun = match_pcre2_regex;
1806 break;
1807 #endif
1808 #ifdef HAVE_LIBPCRE
1809 case regex_type_pcre:
1810 opt.regex_compile_fun = compile_pcre_regex;
1811 opt.regex_match_fun = match_pcre_regex;
1812 break;
1813 #endif
1814
1815 case regex_type_posix:
1816 default:
1817 opt.regex_compile_fun = compile_posix_regex;
1818 opt.regex_match_fun = match_posix_regex;
1819 break;
1820 }
1821 if (opt.acceptregex_s)
1822 {
1823 opt.acceptregex = opt.regex_compile_fun (opt.acceptregex_s);
1824 if (!opt.acceptregex)
1825 exit (WGET_EXIT_GENERIC_ERROR);
1826 }
1827 if (opt.rejectregex_s)
1828 {
1829 opt.rejectregex = opt.regex_compile_fun (opt.rejectregex_s);
1830 if (!opt.rejectregex)
1831 exit (WGET_EXIT_GENERIC_ERROR);
1832 }
1833 if (opt.post_data || opt.post_file_name)
1834 {
1835 if (opt.post_data && opt.post_file_name)
1836 {
1837 fprintf (stderr, _("You cannot specify both --post-data and --post-file.\n"));
1838 exit (WGET_EXIT_GENERIC_ERROR);
1839 }
1840 else if (opt.method)
1841 {
1842 fprintf (stderr, _("You cannot use --post-data or --post-file along with --method. "
1843 "--method expects data through --body-data and --body-file options\n"));
1844 exit (WGET_EXIT_GENERIC_ERROR);
1845 }
1846 }
1847 if (opt.body_data || opt.body_file)
1848 {
1849 if (!opt.method)
1850 {
1851 fprintf (stderr, _("You must specify a method through --method=HTTPMethod "
1852 "to use with --body-data or --body-file.\n"));
1853 exit (WGET_EXIT_GENERIC_ERROR);
1854 }
1855 else if (opt.body_data && opt.body_file)
1856 {
1857 fprintf (stderr, _("You cannot specify both --body-data and --body-file.\n"));
1858 exit (WGET_EXIT_GENERIC_ERROR);
1859 }
1860 }
1861
1862 /* Set various options as required for opt.method. */
1863
1864 /* When user specifies HEAD as the method, we do not wish to download any
1865 files. Hence, set wget to run in spider mode. */
1866 if (opt.method && c_strcasecmp (opt.method, "HEAD") == 0)
1867 setoptval ("spider", "1", "spider");
1868
1869 /* Convert post_data to body-data and post_file_name to body-file options.
1870 This is required so as to remove redundant code later on in gethttp().
1871 The --post-data and --post-file options may also be removed in
1872 the future hence it makes sense to convert them to aliases for
1873 the more generic --method options.
1874 This MUST occur only after the sanity checks so as to prevent the
1875 user from setting both post and body options simultaneously.
1876 */
1877 if (opt.post_data || opt.post_file_name)
1878 {
1879 setoptval ("method", "POST", "method");
1880 if (opt.post_data)
1881 {
1882 setoptval ("bodydata", opt.post_data, "body-data");
1883 xfree(opt.post_data);
1884 }
1885 else
1886 {
1887 setoptval ("bodyfile", opt.post_file_name, "body-file");
1888 xfree(opt.post_file_name);
1889 }
1890 }
1891
1892 #ifdef ENABLE_IRI
1893 if (opt.enable_iri)
1894 {
1895 if (opt.locale && !check_encoding_name (opt.locale))
1896 xfree (opt.locale);
1897
1898 if (!opt.locale)
1899 opt.locale = find_locale ();
1900
1901 if (opt.encoding_remote && !check_encoding_name (opt.encoding_remote))
1902 xfree (opt.encoding_remote);
1903 }
1904 #else
1905 memset (&dummy_iri, 0, sizeof (dummy_iri));
1906 if (opt.enable_iri || opt.locale || opt.encoding_remote)
1907 {
1908 /* sXXXav : be more specific... */
1909 fprintf (stderr, _("This version does not have support for IRIs\n"));
1910 exit (WGET_EXIT_GENERIC_ERROR);
1911 }
1912 #endif
1913
1914 if (opt.ask_passwd)
1915 {
1916 opt.passwd = prompt_for_password ();
1917
1918 if (opt.passwd == NULL || opt.passwd[0] == '\0')
1919 exit (WGET_EXIT_GENERIC_ERROR);
1920 }
1921
1922 if (opt.use_askpass)
1923 {
1924 if (opt.use_askpass[0] == '\0')
1925 {
1926 fprintf (stderr,
1927 _("use-askpass requires a string or either environment variable WGET_ASKPASS or SSH_ASKPASS to be set.\n"));
1928 exit (WGET_EXIT_GENERIC_ERROR);
1929 }
1930 }
1931
1932 #ifdef USE_WATT32
1933 if (opt.wdebug)
1934 dbug_init();
1935 sock_init();
1936 #elif ! defined TESTING
1937 if (opt.background)
1938 {
1939 bool logfile_changed = fork_to_background ();
1940
1941 if (logfile_changed)
1942 log_init (opt.lfilename, append_to_log);
1943 }
1944 #endif
1945
1946 /* Initialize progress. Have to do this after the options are
1947 processed so we know where the log file is. */
1948 if (opt.show_progress)
1949 set_progress_implementation (opt.progress_type);
1950
1951 /* Open WARC file. */
1952 if (opt.warc_filename != 0)
1953 warc_init ();
1954
1955 DEBUGP (("DEBUG output created by Wget %s on %s.\n\n",
1956 version_string, OS_TYPE));
1957
1958 /* Open the output filename if necessary. */
1959
1960 /* 2005-04-17 SMS.
1961 Note that having the output_stream ("-O") file opened here for an FTP
1962 URL rather than in getftp() (ftp.c) (and the http equivalent) rather
1963 limits the ability in VMS to open the file differently for ASCII
1964 versus binary FTP there. (Of course, doing it here allows a open
1965 failure to be detected immediately, without first connecting to the
1966 server.)
1967 */
1968 if (opt.output_document)
1969 {
1970 if (HYPHENP (opt.output_document))
1971 {
1972 #ifdef WINDOWS
1973 _setmode (_fileno (stdout), _O_BINARY);
1974 #endif
1975 output_stream = stdout;
1976 }
1977 else
1978 {
1979 struct stat st;
1980
1981 #ifdef __VMS
1982 /* Common fopen() optional arguments:
1983 sequential access only, access callback function.
1984 */
1985 # define FOPEN_OPT_ARGS , "fop=sqo", "acc", acc_cb, &open_id
1986 int open_id = 7;
1987 #else /* def __VMS */
1988 # define FOPEN_OPT_ARGS
1989 #endif /* def __VMS [else] */
1990
1991 if (opt.unlink_requested)
1992 {
1993 unlink(opt.output_document);
1994 }
1995
1996 output_stream = fopen (opt.output_document,
1997 opt.always_rest ? "ab" : "wb"
1998 FOPEN_OPT_ARGS);
1999 if (output_stream == NULL)
2000 {
2001 perror (opt.output_document);
2002 exit (WGET_EXIT_GENERIC_ERROR);
2003 }
2004 if (fstat (fileno (output_stream), &st) == 0 && S_ISREG (st.st_mode))
2005 output_stream_regular = true;
2006 }
2007 if (!output_stream_regular && (opt.convert_links || opt.recursive))
2008 {
2009 fprintf (stderr, _("-k or -r can be used together with -O only if \
2010 outputting to a regular file.\n"));
2011 exit (WGET_EXIT_GENERIC_ERROR);
2012 }
2013 if (!output_stream_regular && (opt.convert_links || opt.convert_file_only))
2014 {
2015 fprintf (stderr, _("--convert-links or --convert-file-only can be used together \
2016 only if outputting to a regular file.\n"));
2017 exit (WGET_EXIT_GENERIC_ERROR);
2018 }
2019 }
2020
2021 #ifdef HAVE_LIBCARES
2022 if (opt.bind_dns_address || opt.dns_servers)
2023 {
2024 if (ares_library_init (ARES_LIB_INIT_ALL))
2025 {
2026 fprintf (stderr, _("Failed to init libcares\n"));
2027 exit (WGET_EXIT_GENERIC_ERROR);
2028 }
2029
2030 if (ares_init (&ares) != ARES_SUCCESS)
2031 {
2032 fprintf (stderr, _("Failed to init c-ares channel\n"));
2033 exit (WGET_EXIT_GENERIC_ERROR);
2034 }
2035
2036 if (opt.bind_dns_address)
2037 {
2038 struct in_addr a4;
2039 #ifdef ENABLE_IPV6
2040 struct in6_addr a6;
2041 #endif
2042
2043 if (inet_pton (AF_INET, opt.bind_dns_address, &a4) == 1)
2044 {
2045 ares_set_local_ip4 (ares, ntohl (a4.s_addr));
2046 }
2047 #ifdef ENABLE_IPV6
2048 else if (inet_pton (AF_INET6, opt.bind_dns_address, &a6) == 1)
2049 {
2050 ares_set_local_ip6 (ares, (unsigned char *) &a6);
2051 }
2052 #endif
2053 else
2054 {
2055 fprintf (stderr, _("Failed to parse IP address '%s'\n"), opt.bind_dns_address);
2056 exit (WGET_EXIT_GENERIC_ERROR);
2057 }
2058 }
2059
2060 if (opt.dns_servers)
2061 {
2062 int result;
2063
2064 if ((result = ares_set_servers_csv (ares, opt.dns_servers)) != ARES_SUCCESS)
2065 {
2066 fprintf (stderr, _("Failed to set DNS server(s) '%s' (%d)\n"), opt.dns_servers, result);
2067 exit (WGET_EXIT_GENERIC_ERROR);
2068 }
2069 }
2070 }
2071 #endif
2072
2073 #ifdef __VMS
2074 /* Set global ODS5 flag according to the specified destination (if
2075 any), otherwise according to the current default device.
2076 */
2077 if (output_stream == NULL)
2078 set_ods5_dest( "SYS$DISK");
2079 else if (output_stream != stdout)
2080 set_ods5_dest( opt.output_document);
2081 #endif /* def __VMS */
2082
2083 #ifdef WINDOWS
2084 ws_startup ();
2085 #endif
2086
2087 #ifdef SIGHUP
2088 /* Setup the signal handler to redirect output when hangup is
2089 received. */
2090 if (signal(SIGHUP, SIG_IGN) != SIG_IGN)
2091 signal(SIGHUP, redirect_output_signal);
2092 #endif
2093 /* ...and do the same for SIGUSR1. */
2094 #ifdef SIGUSR1
2095 signal (SIGUSR1, redirect_output_signal);
2096 #endif
2097 #ifdef SIGPIPE
2098 /* Writing to a closed socket normally signals SIGPIPE, and the
2099 process exits. What we want is to ignore SIGPIPE and just check
2100 for the return value of write(). */
2101 signal (SIGPIPE, SIG_IGN);
2102 #endif
2103 #ifdef SIGWINCH
2104 signal (SIGWINCH, progress_handle_sigwinch);
2105 #endif
2106
2107 #ifdef HAVE_HSTS
2108 /* Load the HSTS database.
2109 Maybe all the URLs are FTP(S), in which case HSTS would not be needed,
2110 but this is the best place to do it, and it shouldn't be a critical
2111 performance hit.
2112 */
2113 if (opt.hsts)
2114 load_hsts ();
2115 #endif
2116
2117 /* Retrieve the URLs from argument list. */
2118 for (i = 0; i < nurls; i++, optind++)
2119 {
2120 char *t;
2121 char *filename = NULL, *redirected_URL = NULL;
2122 int dt = 0, url_err;
2123 /* Need to do a new struct iri every time, because
2124 * retrieve_url may modify it in some circumstances,
2125 * currently. */
2126 struct iri *iri = iri_new ();
2127 struct url *url_parsed;
2128
2129 t = maybe_prepend_scheme (argv[optind]);
2130 if (!t)
2131 t = argv[optind];
2132
2133 set_uri_encoding (iri, opt.locale, true);
2134 url_parsed = url_parse (t, &url_err, iri, true);
2135
2136 if (!url_parsed)
2137 {
2138 logprintf (LOG_NOTQUIET, "%s: %s.\n", t, url_error (url_err));
2139 inform_exit_status (URLERROR);
2140 }
2141 else
2142 {
2143 /* Request credentials if use_askpass is set. */
2144 if (opt.use_askpass)
2145 use_askpass (url_parsed);
2146
2147 if ((opt.recursive || opt.page_requisites)
2148 && ((url_scheme (t) != SCHEME_FTP
2149 #ifdef HAVE_SSL
2150 && url_scheme (t) != SCHEME_FTPS
2151 #endif
2152 )
2153 || url_uses_proxy (url_parsed)))
2154 {
2155 int old_follow_ftp = opt.follow_ftp;
2156
2157 /* Turn opt.follow_ftp on in case of recursive FTP retrieval */
2158 if (url_scheme (t) == SCHEME_FTP
2159 #ifdef HAVE_SSL
2160 || url_scheme (t) == SCHEME_FTPS
2161 #endif
2162 )
2163 opt.follow_ftp = 1;
2164
2165 retrieve_tree (url_parsed, NULL);
2166
2167 opt.follow_ftp = old_follow_ftp;
2168 }
2169 else
2170 {
2171 retrieve_url (url_parsed, t, &filename, &redirected_URL, NULL,
2172 &dt, opt.recursive, iri, true);
2173 }
2174
2175 if (opt.delete_after && filename != NULL && file_exists_p (filename, NULL))
2176 {
2177 DEBUGP (("Removing file due to --delete-after in main():\n"));
2178 logprintf (LOG_VERBOSE, _("Removing %s.\n"), filename);
2179 if (unlink (filename))
2180 logprintf (LOG_NOTQUIET, "unlink: %s\n", strerror (errno));
2181 }
2182 xfree (redirected_URL);
2183 xfree (filename);
2184 url_free (url_parsed);
2185 }
2186
2187 iri_free (iri);
2188
2189 if (t != argv[optind])
2190 xfree (t);
2191 }
2192
2193 /* And then from the input file, if any. */
2194 if (opt.input_filename)
2195 {
2196 int count;
2197 int status;
2198 status = retrieve_from_file (opt.input_filename, opt.force_html, &count);
2199 inform_exit_status (status);
2200 if (!count)
2201 logprintf (LOG_NOTQUIET, _("No URLs found in %s.\n"),
2202 opt.input_filename);
2203 }
2204
2205 #ifdef HAVE_METALINK
2206 /* Finally, from metlink file, if any. */
2207 if (opt.input_metalink)
2208 {
2209 metalink_error_t meta_err;
2210 uerr_t retr_err;
2211 metalink_t *metalink;
2212
2213 meta_err = metalink_parse_file (opt.input_metalink, &metalink);
2214
2215 if (meta_err)
2216 {
2217 logprintf (LOG_NOTQUIET, _("Unable to parse metalink file %s.\n"),
2218 opt.input_metalink);
2219 retr_err = METALINK_PARSE_ERROR;
2220 }
2221 else
2222 {
2223 /* We need to sort the resources if preferred location
2224 was specified by the user. */
2225 if (opt.preferred_location && opt.preferred_location[0])
2226 {
2227 metalink_file_t **mfile_ptr;
2228 for (mfile_ptr = metalink->files; *mfile_ptr; mfile_ptr++)
2229 {
2230 metalink_resource_t **mres_ptr;
2231 metalink_file_t *mfile = *mfile_ptr;
2232 size_t mres_count = 0;
2233
2234 for (mres_ptr = mfile->resources; *mres_ptr; mres_ptr++)
2235 mres_count++;
2236
2237 stable_sort (mfile->resources,
2238 mres_count,
2239 sizeof (metalink_resource_t *),
2240 metalink_res_cmp);
2241 }
2242 }
2243 retr_err = retrieve_from_metalink (metalink);
2244 if (retr_err != RETROK)
2245 {
2246 logprintf (LOG_NOTQUIET,
2247 _("Could not download all resources from %s.\n"),
2248 quote (opt.input_metalink));
2249 }
2250 metalink_delete (metalink);
2251 }
2252 inform_exit_status (retr_err);
2253 }
2254 #endif /* HAVE_METALINK */
2255
2256 /* Print broken links. */
2257 if (opt.recursive && opt.spider)
2258 print_broken_links ();
2259
2260 /* Print the downloaded sum. */
2261 if ((opt.recursive || opt.page_requisites
2262 || nurls > 1
2263 || (opt.input_filename && total_downloaded_bytes != 0))
2264 &&
2265 total_downloaded_bytes != 0)
2266 {
2267 double end_time = ptimer_measure (timer);
2268 char *wall_time = xstrdup (secs_to_human_time (end_time - start_time));
2269 char *download_time = xstrdup (secs_to_human_time (total_download_time));
2270
2271 ptimer_destroy (timer); timer = NULL;
2272
2273 logprintf (LOG_NOTQUIET,
2274 _("FINISHED --%s--\nTotal wall clock time: %s\n"
2275 "Downloaded: %d files, %s in %s (%s)\n"),
2276 datetime_str (time (NULL)),
2277 wall_time,
2278 numurls,
2279 human_readable (total_downloaded_bytes, 10, 1),
2280 download_time,
2281 retr_rate (total_downloaded_bytes, total_download_time));
2282 xfree (wall_time);
2283 xfree (download_time);
2284
2285 /* Print quota warning, if exceeded. */
2286 if (opt.quota && total_downloaded_bytes > opt.quota)
2287 logprintf (LOG_NOTQUIET,
2288 _("Download quota of %s EXCEEDED!\n"),
2289 human_readable (opt.quota, 10, 1));
2290 }
2291
2292 if (opt.cookies_output)
2293 save_cookies ();
2294
2295 #ifdef HAVE_HSTS
2296 if (opt.hsts && hsts_store)
2297 save_hsts ();
2298 #endif
2299
2300 if ((opt.convert_links || opt.convert_file_only) && !opt.delete_after)
2301 convert_all_links ();
2302
2303 cleanup ();
2304
2305 exit (get_exit_status ());
2306 }
2307
2308 /*
2309 * vim: et ts=2 sw=2
2310 */