Debian Codegraph @

src/ftp.c

raw
1 /* File Transfer Protocol support.
2 Copyright (C) 1996-2011, 2014-2015, 2018-2024 Free Software
3 Foundation, Inc.
4
5 This file is part of GNU Wget.
6
7 GNU Wget is free software; you can redistribute it and/or modify
8 it under the terms of the GNU General Public License as published by
9 the Free Software Foundation; either version 3 of the License, or
10 (at your option) any later version.
11
12 GNU Wget is distributed in the hope that it will be useful,
13 but WITHOUT ANY WARRANTY; without even the implied warranty of
14 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 GNU General Public License for more details.
16
17 You should have received a copy of the GNU General Public License
18 along with Wget. If not, see <http://www.gnu.org/licenses/>.
19
20 Additional permission under GNU GPL version 3 section 7
21
22 If you modify this program, or any covered work, by linking or
23 combining it with the OpenSSL project's OpenSSL library (or a
24 modified version of that library), containing parts covered by the
25 terms of the OpenSSL or SSLeay licenses, the Free Software Foundation
26 grants you additional permission to convey the resulting work.
27 Corresponding Source for a non-source form of such a combination
28 shall include the source code for the parts of OpenSSL used as well
29 as that of the covered work. */
30
31 #include "wget.h"
32
33 #include <stdio.h>
34 #include <stdlib.h>
35 #include <string.h>
36 #include <strings.h>
37 #include <unistd.h>
38 #include <assert.h>
39 #include <errno.h>
40 #include <time.h>
41
42 #include "utils.h"
43 #include "url.h"
44 #include "retr.h"
45 #include "ftp.h"
46 #include "ssl.h"
47 #include "connect.h"
48 #include "host.h"
49 #include "netrc.h"
50 #include "convert.h" /* for downloaded_file */
51 #include "recur.h" /* for INFINITE_RECURSION */
52 #include "warc.h"
53 #include "c-strcase.h"
54 #ifdef ENABLE_XATTR
55 #include "xattr.h"
56 #endif
57
58 #ifdef __VMS
59 # include "vms.h"
60 #endif /* def __VMS */
61
62
63 /* File where the "ls -al" listing will be saved. */
64 #ifdef MSDOS
65 #define LIST_FILENAME "_listing"
66 #else
67 #define LIST_FILENAME ".listing"
68 #endif
69
70 typedef struct
71 {
72 int st; /* connection status */
73 int cmd; /* command code */
74 int csock; /* control connection socket */
75 double dltime; /* time of the download in msecs */
76 enum stype rs; /* remote system reported by ftp server */
77 enum ustype rsu; /* when rs is ST_UNIX, here there are more details */
78 char *id; /* initial directory */
79 char *target; /* target file name */
80 struct url *proxy; /* FTWK-style proxy */
81 } ccon;
82
83
84 /* Look for regexp "( *[0-9]+ *byte" (literal parenthesis) anywhere in
85 the string S, and return the number converted to wgint, if found, 0
86 otherwise. */
87 static wgint
88 ftp_expected_bytes (const char *s)
89 {
90 wgint res;
91
92 while (1)
93 {
94 while (*s && *s != '(')
95 ++s;
96 if (!*s)
97 return 0;
98 ++s; /* skip the '(' */
99 res = str_to_wgint (s, (char **) &s, 10);
100 if (!*s)
101 return 0;
102 while (*s && c_isspace (*s))
103 ++s;
104 if (!*s)
105 return 0;
106 if (c_tolower (*s) != 'b')
107 continue;
108 if (c_strncasecmp (s, "byte", 4))
109 continue;
110 else
111 break;
112 }
113 return res;
114 }
115
116 #ifdef ENABLE_IPV6
117 /*
118 * This function sets up a passive data connection with the FTP server.
119 * It is merely a wrapper around ftp_epsv, ftp_lpsv and ftp_pasv.
120 */
121 static uerr_t
122 ftp_do_pasv (int csock, ip_address *addr, int *port)
123 {
124 uerr_t err;
125
126 /* We need to determine the address family and need to call
127 getpeername, so while we're at it, store the address to ADDR.
128 ftp_pasv and ftp_lpsv can simply override it. */
129 if (!socket_ip_address (csock, addr, ENDPOINT_PEER))
130 abort ();
131
132 /* If our control connection is over IPv6, then we first try EPSV and then
133 * LPSV if the former is not supported. If the control connection is over
134 * IPv4, we simply issue the good old PASV request. */
135 switch (addr->family)
136 {
137 case AF_INET:
138 if (!opt.server_response)
139 logputs (LOG_VERBOSE, "==> PASV ... ");
140 err = ftp_pasv (csock, addr, port);
141 break;
142 case AF_INET6:
143 if (!opt.server_response)
144 logputs (LOG_VERBOSE, "==> EPSV ... ");
145 err = ftp_epsv (csock, addr, port);
146
147 /* If EPSV is not supported try LPSV */
148 if (err == FTPNOPASV)
149 {
150 if (!opt.server_response)
151 logputs (LOG_VERBOSE, "==> LPSV ... ");
152 err = ftp_lpsv (csock, addr, port);
153 }
154 break;
155 default:
156 abort ();
157 }
158
159 return err;
160 }
161
162 /*
163 * This function sets up an active data connection with the FTP server.
164 * It is merely a wrapper around ftp_eprt, ftp_lprt and ftp_port.
165 */
166 static uerr_t
167 ftp_do_port (int csock, int *local_sock)
168 {
169 uerr_t err;
170 ip_address cip;
171
172 if (!socket_ip_address (csock, &cip, ENDPOINT_PEER))
173 abort ();
174
175 /* If our control connection is over IPv6, then we first try EPRT and then
176 * LPRT if the former is not supported. If the control connection is over
177 * IPv4, we simply issue the good old PORT request. */
178 switch (cip.family)
179 {
180 case AF_INET:
181 if (!opt.server_response)
182 logputs (LOG_VERBOSE, "==> PORT ... ");
183 err = ftp_port (csock, local_sock);
184 break;
185 case AF_INET6:
186 if (!opt.server_response)
187 logputs (LOG_VERBOSE, "==> EPRT ... ");
188 err = ftp_eprt (csock, local_sock);
189
190 /* If EPRT is not supported try LPRT */
191 if (err == FTPPORTERR)
192 {
193 if (!opt.server_response)
194 logputs (LOG_VERBOSE, "==> LPRT ... ");
195 err = ftp_lprt (csock, local_sock);
196 }
197 break;
198 default:
199 abort ();
200 }
201 return err;
202 }
203 #else
204
205 static uerr_t
206 ftp_do_pasv (int csock, ip_address *addr, int *port)
207 {
208 if (!opt.server_response)
209 logputs (LOG_VERBOSE, "==> PASV ... ");
210 return ftp_pasv (csock, addr, port);
211 }
212
213 static uerr_t
214 ftp_do_port (int csock, int *local_sock)
215 {
216 if (!opt.server_response)
217 logputs (LOG_VERBOSE, "==> PORT ... ");
218 return ftp_port (csock, local_sock);
219 }
220 #endif
221
222 static void
223 print_length (wgint size, wgint start, bool authoritative)
224 {
225 logprintf (LOG_VERBOSE, _("Length: %s"), number_to_static_string (size));
226 if (size >= 1024)
227 logprintf (LOG_VERBOSE, " (%s)", human_readable (size, 10, 1));
228 if (start > 0)
229 {
230 if (size - start >= 1024)
231 logprintf (LOG_VERBOSE, _(", %s (%s) remaining"),
232 number_to_static_string (size - start),
233 human_readable (size - start, 10, 1));
234 else
235 logprintf (LOG_VERBOSE, _(", %s remaining"),
236 number_to_static_string (size - start));
237 }
238 logputs (LOG_VERBOSE, !authoritative ? _(" (unauthoritative)\n") : "\n");
239 }
240
241 static uerr_t ftp_get_listing (struct url *, struct url *, ccon *, struct fileinfo **);
242
243 static uerr_t
244 get_ftp_greeting (int csock, ccon *con)
245 {
246 uerr_t err = 0;
247
248 /* Get the server's greeting */
249 err = ftp_greeting (csock);
250 if (err != FTPOK)
251 {
252 logputs (LOG_NOTQUIET, "Error in server response. Closing.\n");
253 fd_close (csock);
254 con->csock = -1;
255 }
256
257 return err;
258 }
259
260 #ifdef HAVE_SSL
261 static uerr_t
262 init_control_ssl_connection (int csock, struct url *u, bool *using_control_security)
263 {
264 bool using_security = false;
265
266 /* If '--ftps-implicit' was passed, perform the SSL handshake directly,
267 * and do not send an AUTH command.
268 * Otherwise send an AUTH sequence before login,
269 * and perform the SSL handshake if accepted by server.
270 */
271 if (!opt.ftps_implicit && !opt.server_response)
272 logputs (LOG_VERBOSE, "==> AUTH TLS ... ");
273 if (opt.ftps_implicit || ftp_auth (csock, SCHEME_FTPS) == FTPOK)
274 {
275 if (!ssl_connect_wget (csock, u->host, NULL))
276 {
277 fd_close (csock);
278 return CONSSLERR;
279 }
280 else if (!ssl_check_certificate (csock, u->host))
281 {
282 fd_close (csock);
283 return VERIFCERTERR;
284 }
285
286 if (!opt.ftps_implicit && !opt.server_response)
287 logputs (LOG_VERBOSE, " done.\n");
288
289 /* If implicit FTPS was requested, we act as "normal" FTP, but over SSL.
290 * We're not using RFC 2228 commands.
291 */
292 using_security = true;
293 }
294 else
295 {
296 /* The server does not support 'AUTH TLS'.
297 * Check if --ftps-fallback-to-ftp was passed. */
298 if (opt.ftps_fallback_to_ftp)
299 {
300 logputs (LOG_NOTQUIET, "Server does not support AUTH TLS. Falling back to FTP.\n");
301 using_security = false;
302 }
303 else
304 {
305 fd_close (csock);
306 return FTPNOAUTH;
307 }
308 }
309
310 *using_control_security = using_security;
311 return NOCONERROR;
312 }
313 #endif
314
315 /* Retrieves a file with denoted parameters through opening an FTP
316 connection to the server. It always closes the data connection,
317 and closes the control connection in case of error. If warc_tmp
318 is non-NULL, the downloaded data will be written there as well. */
319 static uerr_t
320 getftp (struct url *u, struct url *original_url,
321 wgint passed_expected_bytes, wgint *qtyread,
322 wgint restval, ccon *con, int count, wgint *last_expected_bytes,
323 FILE *warc_tmp)
324 {
325 int csock, dtsock, local_sock, res;
326 uerr_t err = RETROK; /* appease the compiler */
327 FILE *fp = NULL;
328 char *respline, *tms;
329 const char *user, *passwd, *tmrate;
330 int cmd = con->cmd;
331 wgint expected_bytes = 0;
332 bool got_expected_bytes = false;
333 bool rest_failed = false;
334 int flags;
335 wgint rd_size, previous_rd_size = 0;
336 char type_char;
337 bool try_again;
338 bool list_a_used = false;
339 #ifdef HAVE_SSL
340 enum prot_level prot = (opt.ftps_clear_data_connection ? PROT_CLEAR : PROT_PRIVATE);
341 /* these variables tell whether the target server
342 * accepts the security extensions (RFC 2228) or not,
343 * and whether we're actually using any of them
344 * (encryption at the control connection only,
345 * or both at control and data connections) */
346 bool using_control_security = false, using_data_security = false;
347 #endif
348
349 assert (con != NULL);
350 assert (con->target != NULL);
351
352 /* Debug-check of the sanity of the request by making sure that LIST
353 and RETR are never both requested (since we can handle only one
354 at a time. */
355 assert (!((cmd & DO_LIST) && (cmd & DO_RETR)));
356 /* Make sure that at least *something* is requested. */
357 assert ((cmd & (DO_LIST | DO_CWD | DO_RETR | DO_LOGIN)) != 0);
358
359 *qtyread = restval;
360
361 /* Find the username with priority */
362 if (u->user)
363 user = u->user;
364 else if (opt.user && (opt.use_askpass || opt.ask_passwd))
365 user = opt.user;
366 else if (opt.ftp_user)
367 user = opt.ftp_user;
368 else if (opt.user)
369 user = opt.user;
370 else
371 user = NULL;
372
373 /* Find the password with priority */
374 if (u->passwd)
375 passwd = u->passwd;
376 else if (opt.passwd && (opt.use_askpass || opt.ask_passwd))
377 passwd = opt.passwd;
378 else if (opt.ftp_passwd)
379 passwd = opt.ftp_passwd;
380 else if (opt.passwd)
381 passwd = opt.passwd;
382 else
383 passwd = NULL;
384
385 /* Check for ~/.netrc if none of the above match */
386 if (opt.netrc && (!user || !passwd))
387 search_netrc (u->host, (const char **) &user, (const char **) &passwd, 1, NULL);
388
389 if (!user) user = "anonymous";
390 if (!passwd) passwd = "-wget@";
391
392 dtsock = -1;
393 local_sock = -1;
394 con->dltime = 0;
395
396 #ifdef HAVE_SSL
397 if (u->scheme == SCHEME_FTPS)
398 {
399 /* Initialize SSL layer first */
400 if (!ssl_init ())
401 {
402 scheme_disable (SCHEME_FTPS);
403 logprintf (LOG_NOTQUIET, _("Could not initialize SSL. It will be disabled.\n"));
404 err = SSLINITFAILED;
405 return err;
406 }
407
408 /* If we're using the default FTP port and implicit FTPS was requested,
409 * rewrite the port to the default *implicit* FTPS port.
410 */
411 if (opt.ftps_implicit && u->port == DEFAULT_FTP_PORT)
412 {
413 DEBUGP (("Implicit FTPS was specified. Rewriting default port to %d.\n", DEFAULT_FTPS_IMPLICIT_PORT));
414 u->port = DEFAULT_FTPS_IMPLICIT_PORT;
415 }
416 }
417 #endif
418
419 if (!(cmd & DO_LOGIN))
420 {
421 csock = con->csock;
422 #ifdef HAVE_SSL
423 using_data_security = con->st & DATA_CHANNEL_SECURITY;
424 #endif
425 }
426 else /* cmd & DO_LOGIN */
427 {
428 char *host = con->proxy ? con->proxy->host : u->host;
429 int port = con->proxy ? con->proxy->port : u->port;
430
431 /* Login to the server: */
432
433 /* First: Establish the control connection. */
434
435 csock = connect_to_host (host, port);
436 if (csock == E_HOST)
437 return HOSTERR;
438 else if (csock < 0)
439 return (retryable_socket_connect_error (errno)
440 ? CONERROR : CONIMPOSSIBLE);
441
442 if (cmd & LEAVE_PENDING)
443 con->csock = csock;
444 else
445 con->csock = -1;
446
447 #ifdef HAVE_SSL
448 if (u->scheme == SCHEME_FTPS)
449 {
450 /* If we're in implicit FTPS mode, we have to set up SSL/TLS before everything else.
451 * Otherwise we first read the server's greeting, and then send an "AUTH TLS".
452 */
453 if (opt.ftps_implicit)
454 {
455 err = init_control_ssl_connection (csock, u, &using_control_security);
456 if (err != NOCONERROR)
457 return err;
458 err = get_ftp_greeting (csock, con);
459 if (err != FTPOK)
460 return err;
461 }
462 else
463 {
464 err = get_ftp_greeting (csock, con);
465 if (err != FTPOK)
466 return err;
467 err = init_control_ssl_connection (csock, u, &using_control_security);
468 if (err != NOCONERROR)
469 return err;
470 }
471 }
472 else
473 {
474 err = get_ftp_greeting (csock, con);
475 if (err != FTPOK)
476 return err;
477 }
478 #else
479 err = get_ftp_greeting (csock, con);
480 if (err != FTPOK)
481 return err;
482 #endif
483
484 /* Second: Login with proper USER/PASS sequence. */
485 logprintf (LOG_VERBOSE, _("Logging in as %s ... "),
486 quotearg_style (escape_quoting_style, user));
487 if (opt.server_response)
488 logputs (LOG_ALWAYS, "\n");
489 if (con->proxy)
490 {
491 /* If proxy is in use, log in as username@target-site. */
492 char *logname = concat_strings (user, "@", u->host, (char *) 0);
493 err = ftp_login (csock, logname, passwd);
494 xfree (logname);
495 }
496 else
497 err = ftp_login (csock, user, passwd);
498
499 /* FTPRERR, FTPSRVERR, WRITEFAILED, FTPLOGREFUSED, FTPLOGINC */
500 switch (err)
501 {
502 case FTPRERR:
503 logputs (LOG_VERBOSE, "\n");
504 logputs (LOG_NOTQUIET, _("\
505 Error in server response, closing control connection.\n"));
506 fd_close (csock);
507 con->csock = -1;
508 return err;
509 case FTPSRVERR:
510 logputs (LOG_VERBOSE, "\n");
511 logputs (LOG_NOTQUIET, _("Error in server greeting.\n"));
512 fd_close (csock);
513 con->csock = -1;
514 return err;
515 case WRITEFAILED:
516 logputs (LOG_VERBOSE, "\n");
517 logputs (LOG_NOTQUIET,
518 _("Write failed, closing control connection.\n"));
519 fd_close (csock);
520 con->csock = -1;
521 return err;
522 case FTPLOGREFUSED:
523 logputs (LOG_VERBOSE, "\n");
524 logputs (LOG_NOTQUIET, _("The server refuses login.\n"));
525 fd_close (csock);
526 con->csock = -1;
527 return FTPLOGREFUSED;
528 case FTPLOGINC:
529 logputs (LOG_VERBOSE, "\n");
530 logputs (LOG_NOTQUIET, _("Login incorrect.\n"));
531 fd_close (csock);
532 con->csock = -1;
533 return FTPLOGINC;
534 case FTPOK:
535 if (!opt.server_response)
536 logputs (LOG_VERBOSE, _("Logged in!\n"));
537 break;
538 default:
539 abort ();
540 }
541
542 #ifdef HAVE_SSL
543 if (using_control_security)
544 {
545 /* Send the PBSZ and PROT commands, in that order.
546 * If we are here it means that the server has already accepted
547 * some form of FTPS. Thus, these commands must work.
548 * If they don't work, that's an error. There's no sense in honoring
549 * --ftps-fallback-to-ftp or similar options. */
550 if (u->scheme == SCHEME_FTPS)
551 {
552 if (!opt.server_response)
553 logputs (LOG_VERBOSE, "==> PBSZ 0 ... ");
554 if ((err = ftp_pbsz (csock, 0)) == FTPNOPBSZ)
555 {
556 logputs (LOG_NOTQUIET, _("Server did not accept the 'PBSZ 0' command.\n"));
557 return err;
558 }
559 if (!opt.server_response)
560 logputs (LOG_VERBOSE, "done.");
561
562 if (!opt.server_response)
563 logprintf (LOG_VERBOSE, " ==> PROT %c ... ", (int) prot);
564 if ((err = ftp_prot (csock, prot)) == FTPNOPROT)
565 {
566 logprintf (LOG_NOTQUIET, _("Server did not accept the 'PROT %c' command.\n"), (int) prot);
567 return err;
568 }
569 if (!opt.server_response)
570 logputs (LOG_VERBOSE, "done.\n");
571
572 if (prot != PROT_CLEAR)
573 {
574 using_data_security = true;
575 con->st |= DATA_CHANNEL_SECURITY;
576 }
577 }
578 }
579 #endif
580
581 /* Third: Get the system type */
582 if (!opt.server_response)
583 logprintf (LOG_VERBOSE, "==> SYST ... ");
584 err = ftp_syst (csock, &con->rs, &con->rsu);
585 /* FTPRERR */
586 switch (err)
587 {
588 case FTPRERR:
589 logputs (LOG_VERBOSE, "\n");
590 logputs (LOG_NOTQUIET, _("\
591 Error in server response, closing control connection.\n"));
592 fd_close (csock);
593 con->csock = -1;
594 return err;
595 case FTPSRVERR:
596 logputs (LOG_VERBOSE, "\n");
597 logputs (LOG_NOTQUIET,
598 _("Server error, can't determine system type.\n"));
599 break;
600 case FTPOK:
601 /* Everything is OK. */
602 break;
603 default:
604 abort ();
605 }
606 if (!opt.server_response && err != FTPSRVERR)
607 logputs (LOG_VERBOSE, _("done. "));
608
609 /* 2013-10-17 Andrea Urbani (matfanjol)
610 According to the system type I choose which
611 list command will be used.
612 If I don't know that system, I will try, the
613 first time of each session, "LIST -a" and
614 "LIST". (see __LIST_A_EXPLANATION__ below) */
615 switch (con->rs)
616 {
617 case ST_VMS:
618 /* About ST_VMS there is an old note:
619 2008-01-29 SMS. For a VMS FTP server, where "LIST -a" may not
620 fail, but will never do what is desired here,
621 skip directly to the simple "LIST" command
622 (assumed to be the last one in the list). */
623 DEBUGP (("\nVMS: I know it and I will use \"LIST\" as standard list command\n"));
624 con->st |= LIST_AFTER_LIST_A_CHECK_DONE;
625 con->st |= AVOID_LIST_A;
626 break;
627 case ST_UNIX:
628 if (con->rsu == UST_MULTINET)
629 {
630 DEBUGP (("\nUNIX MultiNet: I know it and I will use \"LIST\" "
631 "as standard list command\n"));
632 con->st |= LIST_AFTER_LIST_A_CHECK_DONE;
633 con->st |= AVOID_LIST_A;
634 }
635 else if (con->rsu == UST_TYPE_L8)
636 {
637 DEBUGP (("\nUNIX TYPE L8: I know it and I will use \"LIST -a\" "
638 "as standard list command\n"));
639 con->st |= LIST_AFTER_LIST_A_CHECK_DONE;
640 con->st |= AVOID_LIST;
641 }
642 break;
643 default:
644 break;
645 }
646
647 /* Fourth: Find the initial ftp directory */
648
649 if (!opt.server_response)
650 logprintf (LOG_VERBOSE, "==> PWD ... ");
651 err = ftp_pwd (csock, &con->id);
652 /* FTPRERR */
653 switch (err)
654 {
655 case FTPRERR:
656 logputs (LOG_VERBOSE, "\n");
657 logputs (LOG_NOTQUIET, _("\
658 Error in server response, closing control connection.\n"));
659 fd_close (csock);
660 con->csock = -1;
661 return err;
662 case FTPSRVERR :
663 /* PWD unsupported -- assume "/". */
664 xfree (con->id);
665 con->id = xstrdup ("/");
666 break;
667 case FTPOK:
668 /* Everything is OK. */
669 break;
670 default:
671 abort ();
672 }
673
674 #if 0
675 /* 2004-09-17 SMS.
676 Don't help me out. Please.
677 A reasonably recent VMS FTP server will cope just fine with
678 UNIX file specifications. This code just spoils things.
679 Discarding the device name, for example, is not a wise move.
680 This code was disabled but left in as an example of what not
681 to do.
682 */
683
684 /* VMS will report something like "PUB$DEVICE:[INITIAL.FOLDER]".
685 Convert it to "/INITIAL/FOLDER" */
686 if (con->rs == ST_VMS)
687 {
688 char *path = strchr (con->id, '[');
689 char *pathend = path ? strchr (path + 1, ']') : NULL;
690 if (!path || !pathend)
691 DEBUGP (("Initial VMS directory not in the form [...]!\n"));
692 else
693 {
694 char *idir = con->id;
695 DEBUGP (("Preprocessing the initial VMS directory\n"));
696 DEBUGP ((" old = '%s'\n", con->id));
697 /* We do the conversion in-place by copying the stuff
698 between [ and ] to the beginning, and changing dots
699 to slashes at the same time. */
700 *idir++ = '/';
701 for (++path; path < pathend; path++, idir++)
702 *idir = *path == '.' ? '/' : *path;
703 *idir = '\0';
704 DEBUGP ((" new = '%s'\n\n", con->id));
705 }
706 }
707 #endif /* 0 */
708
709 if (!opt.server_response)
710 logputs (LOG_VERBOSE, _("done.\n"));
711
712 /* Fifth: Set the FTP type. */
713 type_char = ftp_process_type (u->params);
714 if (!opt.server_response)
715 logprintf (LOG_VERBOSE, "==> TYPE %c ... ", type_char);
716 err = ftp_type (csock, type_char);
717 /* FTPRERR, WRITEFAILED, FTPUNKNOWNTYPE */
718 switch (err)
719 {
720 case FTPRERR:
721 logputs (LOG_VERBOSE, "\n");
722 logputs (LOG_NOTQUIET, _("\
723 Error in server response, closing control connection.\n"));
724 fd_close (csock);
725 con->csock = -1;
726 return err;
727 case WRITEFAILED:
728 logputs (LOG_VERBOSE, "\n");
729 logputs (LOG_NOTQUIET,
730 _("Write failed, closing control connection.\n"));
731 fd_close (csock);
732 con->csock = -1;
733 return err;
734 case FTPUNKNOWNTYPE:
735 logputs (LOG_VERBOSE, "\n");
736 logprintf (LOG_NOTQUIET,
737 _("Unknown type `%c', closing control connection.\n"),
738 type_char);
739 fd_close (csock);
740 con->csock = -1;
741 return err;
742 case FTPOK:
743 /* Everything is OK. */
744 break;
745 default:
746 abort ();
747 }
748 if (!opt.server_response)
749 logputs (LOG_VERBOSE, _("done. "));
750 } /* do login */
751
752 if (cmd & DO_CWD)
753 {
754 if (!*u->dir)
755 logputs (LOG_VERBOSE, _("==> CWD not needed.\n"));
756 else
757 {
758 const char *targ = NULL;
759 char *target = u->dir;
760 char targetbuf[1024];
761 int cwd_count;
762 int cwd_end;
763 int cwd_start;
764
765 DEBUGP (("changing working directory\n"));
766
767 /* Change working directory. To change to a non-absolute
768 Unix directory, we need to prepend initial directory
769 (con->id) to it. Absolute directories "just work".
770
771 A relative directory is one that does not begin with '/'
772 and, on non-Unix OS'es, one that doesn't begin with
773 "[a-z]:".
774
775 This is not done for OS400, which doesn't use
776 "/"-delimited directories, nor does it support directory
777 hierarchies. "CWD foo" followed by "CWD bar" leaves us
778 in "bar", not in "foo/bar", as would be customary
779 elsewhere. */
780
781 /* 2004-09-20 SMS.
782 Why is this wise even on UNIX? It certainly fouls VMS.
783 See below for a more reliable, more universal method.
784 */
785
786 /* 2008-04-22 MJC.
787 I'm not crazy about it either. I'm informed it's useful
788 for misconfigured servers that have some dirs in the path
789 with +x but -r, but this method is not RFC-conformant. I
790 understand the need to deal with crappy server
791 configurations, but it's far better to use the canonical
792 method first, and fall back to kludges second.
793 */
794
795 if (target[0] != '/'
796 && !(con->rs != ST_UNIX
797 && c_isalpha (target[0])
798 && target[1] == ':')
799 && (con->rs != ST_OS400)
800 && (con->rs != ST_VMS))
801 {
802 char *ntarget, *p;
803 size_t idlen = strlen (con->id);
804 size_t len;
805
806 /* Strip trailing slash(es) from con->id. */
807 while (idlen > 0 && con->id[idlen - 1] == '/')
808 --idlen;
809
810 len = idlen + 1 + strlen (target);
811 if (len < sizeof (targetbuf))
812 p = ntarget = targetbuf;
813 else
814 p = ntarget = xmalloc (len + 1);
815
816 memcpy (p, con->id, idlen);
817 p += idlen;
818 *p++ = '/';
819 strcpy (p, target);
820
821 DEBUGP (("Prepended initial PWD to relative path:\n"));
822 DEBUGP ((" pwd: '%s'\n old: '%s'\n new: '%s'\n",
823 con->id, target, ntarget));
824 target = ntarget;
825 }
826
827 #if 0
828 /* 2004-09-17 SMS.
829 Don't help me out. Please.
830 A reasonably recent VMS FTP server will cope just fine with
831 UNIX file specifications. This code just spoils things.
832 Discarding the device name, for example, is not a wise
833 move.
834 This code was disabled but left in as an example of what
835 not to do.
836 */
837
838 /* If the FTP host runs VMS, we will have to convert the absolute
839 directory path in UNIX notation to absolute directory path in
840 VMS notation as VMS FTP servers do not like UNIX notation of
841 absolute paths. "VMS notation" is [dir.subdir.subsubdir]. */
842
843 if (con->rs == ST_VMS)
844 {
845 char *tmpp;
846 char *ntarget = alloca (strlen (target) + 2);
847 /* We use a converted initial dir, so directories in
848 TARGET will be separated with slashes, something like
849 "/INITIAL/FOLDER/DIR/SUBDIR". Convert that to
850 "[INITIAL.FOLDER.DIR.SUBDIR]". */
851 strcpy (ntarget, target);
852 assert (*ntarget == '/');
853 *ntarget = '[';
854 for (tmpp = ntarget + 1; *tmpp; tmpp++)
855 if (*tmpp == '/')
856 *tmpp = '.';
857 *tmpp++ = ']';
858 *tmpp = '\0';
859 DEBUGP (("Changed file name to VMS syntax:\n"));
860 DEBUGP ((" Unix: '%s'\n VMS: '%s'\n", target, ntarget));
861 target = ntarget;
862 }
863 #endif /* 0 */
864
865 /* 2004-09-20 SMS.
866 A relative directory is relative to the initial directory.
867 Thus, what _is_ useful on VMS (and probably elsewhere) is
868 to CWD to the initial directory (ideally, whatever the
869 server reports, _exactly_, NOT badly UNIX-ixed), and then
870 CWD to the (new) relative directory. This should probably
871 be restructured as a function, called once or twice, but
872 I'm lazy enough to take the badly indented loop short-cut
873 for now.
874 */
875
876 /* Decide on one pass (absolute) or two (relative).
877 The VMS restriction may be relaxed when the squirrely code
878 above is reformed.
879 */
880 if ((con->rs == ST_VMS) && (target[0] != '/'))
881 {
882 cwd_start = 0;
883 DEBUGP (("Using two-step CWD for relative path.\n"));
884 }
885 else
886 {
887 /* Go straight to the target. */
888 cwd_start = 1;
889 }
890
891 /* At least one VMS FTP server (TCPware V5.6-2) can switch to
892 a UNIX emulation mode when given a UNIX-like directory
893 specification (like "a/b/c"). If allowed to continue this
894 way, LIST interpretation will be confused, because the
895 system type (SYST response) will not be re-checked, and
896 future UNIX-format directory listings (for multiple URLs or
897 "-r") will be horribly misinterpreted.
898
899 The cheap and nasty work-around is to do a "CWD []" after a
900 UNIX-like directory specification is used. (A single-level
901 directory is harmless.) This puts the TCPware server back
902 into VMS mode, and does no harm on other servers.
903
904 Unlike the rest of this block, this particular behavior
905 _is_ VMS-specific, so it gets its own VMS test.
906 */
907 if ((con->rs == ST_VMS) && (strchr (target, '/') != NULL))
908 {
909 cwd_end = 3;
910 DEBUGP (("Using extra \"CWD []\" step for VMS server.\n"));
911 }
912 else
913 {
914 cwd_end = 2;
915 }
916
917 /* 2004-09-20 SMS. */
918 /* Sorry about the deviant indenting. Laziness. */
919
920 for (cwd_count = cwd_start; cwd_count < cwd_end; cwd_count++)
921 {
922 switch (cwd_count)
923 {
924 case 0:
925 /* Step one (optional): Go to the initial directory,
926 exactly as reported by the server.
927 */
928 targ = con->id;
929 break;
930
931 case 1:
932 /* Step two: Go to the target directory. (Absolute or
933 relative will work now.)
934 */
935 targ = target;
936 break;
937
938 case 2:
939 /* Step three (optional): "CWD []" to restore server
940 VMS-ness.
941 */
942 targ = "[]";
943 break;
944
945 default:
946 logprintf (LOG_ALWAYS, _("Logically impossible section reached in getftp()"));
947 logprintf (LOG_ALWAYS, _("cwd_count: %d\ncwd_start: %d\ncwd_end: %d\n"),
948 cwd_count, cwd_start, cwd_end);
949 abort ();
950 }
951
952 if (!opt.server_response)
953 logprintf (LOG_VERBOSE, "==> CWD (%d) %s ... ", cwd_count,
954 quotearg_style (escape_quoting_style, target));
955
956 err = ftp_cwd (csock, targ);
957
958 /* FTPRERR, WRITEFAILED, FTPNSFOD */
959 switch (err)
960 {
961 case FTPRERR:
962 logputs (LOG_VERBOSE, "\n");
963 logputs (LOG_NOTQUIET, _("\
964 Error in server response, closing control connection.\n"));
965 fd_close (csock);
966 con->csock = -1;
967 return err;
968 case WRITEFAILED:
969 logputs (LOG_VERBOSE, "\n");
970 logputs (LOG_NOTQUIET,
971 _("Write failed, closing control connection.\n"));
972 fd_close (csock);
973 con->csock = -1;
974 return err;
975 case FTPNSFOD:
976 logputs (LOG_VERBOSE, "\n");
977 logprintf (LOG_NOTQUIET, _("No such directory %s.\n\n"),
978 quote (u->dir));
979 fd_close (csock);
980 con->csock = -1;
981 return err;
982 case FTPOK:
983 break;
984 default:
985 abort ();
986 }
987
988 if (!opt.server_response)
989 logputs (LOG_VERBOSE, _("done.\n"));
990
991 } /* for */
992
993 /* 2004-09-20 SMS. */
994
995 } /* else */
996 }
997 else /* do not CWD */
998 logputs (LOG_VERBOSE, _("==> CWD not required.\n"));
999
1000 if ((cmd & DO_RETR) && passed_expected_bytes == 0)
1001 {
1002 if (opt.verbose)
1003 {
1004 if (!opt.server_response)
1005 logprintf (LOG_VERBOSE, "==> SIZE %s ... ",
1006 quotearg_style (escape_quoting_style, u->file));
1007 }
1008
1009 err = ftp_size (csock, u->file, &expected_bytes);
1010 /* FTPRERR */
1011 switch (err)
1012 {
1013 case FTPRERR:
1014 case FTPSRVERR:
1015 logputs (LOG_VERBOSE, "\n");
1016 logputs (LOG_NOTQUIET, _("\
1017 Error in server response, closing control connection.\n"));
1018 fd_close (csock);
1019 con->csock = -1;
1020 return err;
1021 case FTPOK:
1022 got_expected_bytes = true;
1023 /* Everything is OK. */
1024 break;
1025 default:
1026 abort ();
1027 }
1028 if (!opt.server_response)
1029 {
1030 logprintf (LOG_VERBOSE, "%s\n",
1031 expected_bytes ?
1032 number_to_static_string (expected_bytes) :
1033 _("done.\n"));
1034 }
1035 }
1036
1037 if (cmd & DO_RETR && restval > 0 && restval == expected_bytes)
1038 {
1039 /* Server confirms that file has length restval. We should stop now.
1040 Some servers (f.e. NcFTPd) return error when receive REST 0 */
1041 logputs (LOG_VERBOSE, _("File has already been retrieved.\n"));
1042 fd_close (csock);
1043 con->csock = -1;
1044 return RETRFINISHED;
1045 }
1046
1047 do
1048 {
1049 try_again = false;
1050 /* If anything is to be retrieved, PORT (or PASV) must be sent. */
1051 if (cmd & (DO_LIST | DO_RETR))
1052 {
1053 if (opt.ftp_pasv)
1054 {
1055 ip_address passive_addr;
1056 int passive_port;
1057 err = ftp_do_pasv (csock, &passive_addr, &passive_port);
1058 /* FTPRERR, WRITEFAILED, FTPNOPASV, FTPINVPASV */
1059 switch (err)
1060 {
1061 case FTPRERR:
1062 logputs (LOG_VERBOSE, "\n");
1063 logputs (LOG_NOTQUIET, _("\
1064 Error in server response, closing control connection.\n"));
1065 fd_close (csock);
1066 con->csock = -1;
1067 return err;
1068 case WRITEFAILED:
1069 logputs (LOG_VERBOSE, "\n");
1070 logputs (LOG_NOTQUIET,
1071 _("Write failed, closing control connection.\n"));
1072 fd_close (csock);
1073 con->csock = -1;
1074 return err;
1075 case FTPNOPASV:
1076 logputs (LOG_VERBOSE, "\n");
1077 logputs (LOG_NOTQUIET, _("Cannot initiate PASV transfer.\n"));
1078 break;
1079 case FTPINVPASV:
1080 logputs (LOG_VERBOSE, "\n");
1081 logputs (LOG_NOTQUIET, _("Cannot parse PASV response.\n"));
1082 break;
1083 case FTPOK:
1084 break;
1085 default:
1086 abort ();
1087 } /* switch (err) */
1088 if (err==FTPOK)
1089 {
1090 DEBUGP (("trying to connect to %s port %d\n",
1091 print_address (&passive_addr), passive_port));
1092 dtsock = connect_to_ip (&passive_addr, passive_port, NULL);
1093 if (dtsock < 0)
1094 {
1095 int save_errno = errno;
1096 fd_close (csock);
1097 con->csock = -1;
1098 logprintf (LOG_VERBOSE, _("couldn't connect to %s port %d: %s\n"),
1099 print_address (&passive_addr), passive_port,
1100 strerror (save_errno));
1101 return (retryable_socket_connect_error (save_errno)
1102 ? CONERROR : CONIMPOSSIBLE);
1103 }
1104
1105 if (!opt.server_response)
1106 logputs (LOG_VERBOSE, _("done. "));
1107 }
1108 else
1109 return err;
1110
1111 /*
1112 * We do not want to fall back from PASSIVE mode to ACTIVE mode !
1113 * The reason is the PORT command exposes the client's real IP address
1114 * to the server. Bad for someone who relies on privacy via a ftp proxy.
1115 */
1116 }
1117 else
1118 {
1119 err = ftp_do_port (csock, &local_sock);
1120 /* FTPRERR, WRITEFAILED, bindport (FTPSYSERR), HOSTERR,
1121 FTPPORTERR */
1122 switch (err)
1123 {
1124 case FTPRERR:
1125 logputs (LOG_VERBOSE, "\n");
1126 logputs (LOG_NOTQUIET, _("\
1127 Error in server response, closing control connection.\n"));
1128 fd_close (csock);
1129 con->csock = -1;
1130 fd_close (dtsock);
1131 fd_close (local_sock);
1132 return err;
1133 case WRITEFAILED:
1134 logputs (LOG_VERBOSE, "\n");
1135 logputs (LOG_NOTQUIET,
1136 _("Write failed, closing control connection.\n"));
1137 fd_close (csock);
1138 con->csock = -1;
1139 fd_close (dtsock);
1140 fd_close (local_sock);
1141 return err;
1142 case CONSOCKERR:
1143 logputs (LOG_VERBOSE, "\n");
1144 logprintf (LOG_NOTQUIET, "socket: %s\n", strerror (errno));
1145 fd_close (csock);
1146 con->csock = -1;
1147 fd_close (dtsock);
1148 fd_close (local_sock);
1149 return err;
1150 case FTPSYSERR:
1151 logputs (LOG_VERBOSE, "\n");
1152 logprintf (LOG_NOTQUIET, _("Bind error (%s).\n"),
1153 strerror (errno));
1154 fd_close (dtsock);
1155 return err;
1156 case FTPPORTERR:
1157 logputs (LOG_VERBOSE, "\n");
1158 logputs (LOG_NOTQUIET, _("Invalid PORT.\n"));
1159 fd_close (csock);
1160 con->csock = -1;
1161 fd_close (dtsock);
1162 fd_close (local_sock);
1163 return err;
1164 case FTPOK:
1165 break;
1166 default:
1167 abort ();
1168 } /* port switch */
1169 if (!opt.server_response)
1170 logputs (LOG_VERBOSE, _("done. "));
1171 } /* dtsock == -1 */
1172 } /* cmd & (DO_LIST | DO_RETR) */
1173
1174 /* Restart if needed. */
1175 if (restval && (cmd & DO_RETR))
1176 {
1177 if (!opt.server_response)
1178 logprintf (LOG_VERBOSE, "==> REST %s ... ",
1179 number_to_static_string (restval));
1180 err = ftp_rest (csock, restval);
1181
1182 /* FTPRERR, WRITEFAILED, FTPRESTFAIL */
1183 switch (err)
1184 {
1185 case FTPRERR:
1186 logputs (LOG_VERBOSE, "\n");
1187 logputs (LOG_NOTQUIET, _("\
1188 Error in server response, closing control connection.\n"));
1189 fd_close (csock);
1190 con->csock = -1;
1191 fd_close (dtsock);
1192 fd_close (local_sock);
1193 return err;
1194 case WRITEFAILED:
1195 logputs (LOG_VERBOSE, "\n");
1196 logputs (LOG_NOTQUIET,
1197 _("Write failed, closing control connection.\n"));
1198 fd_close (csock);
1199 con->csock = -1;
1200 fd_close (dtsock);
1201 fd_close (local_sock);
1202 return err;
1203 case FTPRESTFAIL:
1204 logputs (LOG_VERBOSE, _("\nREST failed, starting from scratch.\n"));
1205 rest_failed = true;
1206 break;
1207 case FTPOK:
1208 break;
1209 default:
1210 abort ();
1211 }
1212 if (err != FTPRESTFAIL && !opt.server_response)
1213 logputs (LOG_VERBOSE, _("done. "));
1214 } /* restval && cmd & DO_RETR */
1215
1216 if (cmd & DO_RETR)
1217 {
1218 /* If we're in spider mode, don't really retrieve anything except
1219 the directory listing and verify whether the given "file" exists. */
1220 if (opt.spider)
1221 {
1222 bool exists = false;
1223 bool all_exist = true;
1224 struct fileinfo *f;
1225 uerr_t _res = ftp_get_listing (u, original_url, con, &f);
1226 /* Set the DO_RETR command flag again, because it gets unset when
1227 calling ftp_get_listing() and would otherwise cause an assertion
1228 failure earlier on when this function gets repeatedly called
1229 (e.g., when recursing). */
1230 con->cmd |= DO_RETR;
1231 if (_res == RETROK)
1232 {
1233 while (f)
1234 {
1235 if (!strcmp (f->name, u->file))
1236 {
1237 exists = true;
1238 break;
1239 } else {
1240 all_exist = false;
1241 }
1242 f = f->next;
1243 }
1244 if (exists)
1245 {
1246 logputs (LOG_VERBOSE, "\n");
1247 logprintf (LOG_NOTQUIET, _("File %s exists.\n"),
1248 quote (u->file));
1249 }
1250 else
1251 {
1252 logputs (LOG_VERBOSE, "\n");
1253 logprintf (LOG_NOTQUIET, _("No such file %s.\n"),
1254 quote (u->file));
1255 }
1256 }
1257 fd_close (csock);
1258 con->csock = -1;
1259 fd_close (dtsock);
1260 fd_close (local_sock);
1261 if (all_exist) {
1262 return RETRFINISHED;
1263 } else {
1264 return FTPNSFOD;
1265 }
1266 }
1267
1268 if (opt.verbose)
1269 {
1270 if (!opt.server_response)
1271 {
1272 if (restval)
1273 logputs (LOG_VERBOSE, "\n");
1274 logprintf (LOG_VERBOSE, "==> RETR %s ... ",
1275 quotearg_style (escape_quoting_style, u->file));
1276 }
1277 }
1278
1279 err = ftp_retr (csock, u->file);
1280 /* FTPRERR, WRITEFAILED, FTPNSFOD */
1281 switch (err)
1282 {
1283 case FTPRERR:
1284 logputs (LOG_VERBOSE, "\n");
1285 logputs (LOG_NOTQUIET, _("\
1286 Error in server response, closing control connection.\n"));
1287 fd_close (csock);
1288 con->csock = -1;
1289 fd_close (dtsock);
1290 fd_close (local_sock);
1291 return err;
1292 case WRITEFAILED:
1293 logputs (LOG_VERBOSE, "\n");
1294 logputs (LOG_NOTQUIET,
1295 _("Write failed, closing control connection.\n"));
1296 fd_close (csock);
1297 con->csock = -1;
1298 fd_close (dtsock);
1299 fd_close (local_sock);
1300 return err;
1301 case FTPNSFOD:
1302 logputs (LOG_VERBOSE, "\n");
1303 logprintf (LOG_NOTQUIET, _("No such file %s.\n\n"),
1304 quote (u->file));
1305 fd_close (dtsock);
1306 fd_close (local_sock);
1307 return err;
1308 case FTPOK:
1309 break;
1310 default:
1311 abort ();
1312 }
1313
1314 if (!opt.server_response)
1315 logputs (LOG_VERBOSE, _("done.\n"));
1316
1317 if (! got_expected_bytes)
1318 expected_bytes = *last_expected_bytes;
1319 } /* do retrieve */
1320
1321 if (cmd & DO_LIST)
1322 {
1323 if (!opt.server_response)
1324 logputs (LOG_VERBOSE, "==> LIST ... ");
1325 /* As Maciej W. Rozycki ([email protected]) says, `LIST'
1326 without arguments is better than `LIST .'; confirmed by
1327 RFC959. */
1328 err = ftp_list (csock, NULL, con->st&AVOID_LIST_A, con->st&AVOID_LIST, &list_a_used);
1329
1330 /* FTPRERR, WRITEFAILED */
1331 switch (err)
1332 {
1333 case FTPRERR:
1334 logputs (LOG_VERBOSE, "\n");
1335 logputs (LOG_NOTQUIET, _("\
1336 Error in server response, closing control connection.\n"));
1337 fd_close (csock);
1338 con->csock = -1;
1339 fd_close (dtsock);
1340 fd_close (local_sock);
1341 return err;
1342 case WRITEFAILED:
1343 logputs (LOG_VERBOSE, "\n");
1344 logputs (LOG_NOTQUIET,
1345 _("Write failed, closing control connection.\n"));
1346 fd_close (csock);
1347 con->csock = -1;
1348 fd_close (dtsock);
1349 fd_close (local_sock);
1350 return err;
1351 case FTPNSFOD:
1352 logputs (LOG_VERBOSE, "\n");
1353 logprintf (LOG_NOTQUIET, _("No such file or directory %s.\n\n"),
1354 quote ("."));
1355 fd_close (dtsock);
1356 fd_close (local_sock);
1357 return err;
1358 case FTPOK:
1359 break;
1360 default:
1361 abort ();
1362 }
1363 if (!opt.server_response)
1364 logputs (LOG_VERBOSE, _("done.\n"));
1365
1366 if (! got_expected_bytes)
1367 expected_bytes = *last_expected_bytes;
1368 } /* cmd & DO_LIST */
1369
1370 if (!(cmd & (DO_LIST | DO_RETR)) || (opt.spider && !(cmd & DO_LIST)))
1371 return RETRFINISHED;
1372
1373 /* Some FTP servers return the total length of file after REST
1374 command, others just return the remaining size. */
1375 if (passed_expected_bytes && restval && expected_bytes
1376 && (expected_bytes == passed_expected_bytes - restval))
1377 {
1378 DEBUGP (("Lying FTP server found, adjusting.\n"));
1379 expected_bytes = passed_expected_bytes;
1380 }
1381
1382 /* If no transmission was required, then everything is OK. */
1383 if (!opt.ftp_pasv) /* we are not using passive mode so we need
1384 to accept */
1385 {
1386 /* Wait for the server to connect to the address we're waiting
1387 at. */
1388 dtsock = accept_connection (local_sock);
1389 if (dtsock < 0)
1390 {
1391 logprintf (LOG_NOTQUIET, "accept: %s\n", strerror (errno));
1392 return CONERROR;
1393 }
1394 }
1395
1396 /* Open the file -- if output_stream is set, use it instead. */
1397
1398 /* 2005-04-17 SMS.
1399 Note that having the output_stream ("-O") file opened in main
1400 (main.c) rather limits the ability in VMS to open the file
1401 differently for ASCII versus binary FTP here. (Of course, doing it
1402 there allows a open failure to be detected immediately, without first
1403 connecting to the server.)
1404 */
1405 if (!output_stream || con->cmd & DO_LIST)
1406 {
1407 /* On VMS, alter the name as required. */
1408 #ifdef __VMS
1409 char *targ;
1410
1411 targ = ods_conform (con->target);
1412 if (targ != con->target)
1413 {
1414 xfree (con->target);
1415 con->target = targ;
1416 }
1417 #endif /* def __VMS */
1418
1419 mkalldirs (con->target);
1420 if (opt.backups)
1421 rotate_backups (con->target);
1422
1423 /* 2005-04-15 SMS.
1424 For VMS, define common fopen() optional arguments, and a handy macro
1425 for use as a variable "binary" flag.
1426 Elsewhere, define a constant "binary" flag.
1427 Isn't it nice to have distinct text and binary file types?
1428 */
1429 /* 2011-09-30 SMS.
1430 Added listing files to the set of non-"binary" (text, Stream_LF)
1431 files. (Wget works either way, but other programs, like, say, text
1432 editors, work better on listing files which have text attributes.)
1433 Now we use "binary" attributes for a binary ("IMAGE") transfer,
1434 unless "--ftp-stmlf" was specified, and we always use non-"binary"
1435 (text, Stream_LF) attributes for a listing file, or for an ASCII
1436 transfer.
1437 Tidied the VMS-specific BIN_TYPE_xxx macros, and changed the call to
1438 fopen_excl() (restored?) to use BIN_TYPE_FILE instead of "true".
1439 */
1440 #ifdef __VMS
1441 # define BIN_TYPE_TRANSFER (type_char != 'A')
1442 # define BIN_TYPE_FILE \
1443 ((!(cmd & DO_LIST)) && BIN_TYPE_TRANSFER && (opt.ftp_stmlf == 0))
1444 # define FOPEN_OPT_ARGS "fop=sqo", "acc", acc_cb, &open_id
1445 # define FOPEN_OPT_ARGS_BIN "ctx=bin,stm", "rfm=fix", "mrs=512" FOPEN_OPT_ARGS
1446 #else /* def __VMS */
1447 # define BIN_TYPE_FILE true
1448 #endif /* def __VMS [else] */
1449
1450 if (restval && !(con->cmd & DO_LIST))
1451 {
1452 #ifdef __VMS
1453 int open_id;
1454
1455 if (BIN_TYPE_FILE)
1456 {
1457 open_id = 3;
1458 fp = fopen (con->target, "ab", FOPEN_OPT_ARGS_BIN);
1459 }
1460 else
1461 {
1462 open_id = 4;
1463 fp = fopen (con->target, "a", FOPEN_OPT_ARGS);
1464 }
1465 #else /* def __VMS */
1466 fp = fopen (con->target, "ab");
1467 #endif /* def __VMS [else] */
1468 }
1469 else if (opt.noclobber || opt.always_rest || opt.timestamping || opt.dirstruct
1470 || opt.output_document || count > 0)
1471 {
1472 if (opt.unlink_requested && file_exists_p (con->target, NULL))
1473 {
1474 if (unlink (con->target) < 0)
1475 {
1476 logprintf (LOG_NOTQUIET, "%s: %s\n", con->target,
1477 strerror (errno));
1478 fd_close (csock);
1479 con->csock = -1;
1480 fd_close (dtsock);
1481 fd_close (local_sock);
1482 return UNLINKERR;
1483 }
1484 }
1485
1486 #ifdef __VMS
1487 int open_id;
1488
1489 if (BIN_TYPE_FILE)
1490 {
1491 open_id = 5;
1492 fp = fopen (con->target, "wb", FOPEN_OPT_ARGS_BIN);
1493 }
1494 else
1495 {
1496 open_id = 6;
1497 fp = fopen (con->target, "w", FOPEN_OPT_ARGS);
1498 }
1499 #else /* def __VMS */
1500 fp = fopen (con->target, "wb");
1501 #endif /* def __VMS [else] */
1502 }
1503 else
1504 {
1505 fp = fopen_excl (con->target, BIN_TYPE_FILE);
1506 if (!fp && errno == EEXIST)
1507 {
1508 /* We cannot just invent a new name and use it (which is
1509 what functions like unique_create typically do)
1510 because we told the user we'd use this name.
1511 Instead, return and retry the download. */
1512 logprintf (LOG_NOTQUIET, _("%s has sprung into existence.\n"),
1513 con->target);
1514 fd_close (csock);
1515 con->csock = -1;
1516 fd_close (dtsock);
1517 fd_close (local_sock);
1518 return FOPEN_EXCL_ERR;
1519 }
1520 }
1521 if (!fp)
1522 {
1523 logprintf (LOG_NOTQUIET, "%s: %s\n", con->target, strerror (errno));
1524 fd_close (csock);
1525 con->csock = -1;
1526 fd_close (dtsock);
1527 fd_close (local_sock);
1528 return FOPENERR;
1529 }
1530 }
1531 else
1532 fp = output_stream;
1533
1534 if (passed_expected_bytes)
1535 {
1536 print_length (passed_expected_bytes, restval, true);
1537 expected_bytes = passed_expected_bytes;
1538 /* for fd_read_body's progress bar */
1539 }
1540 else if (expected_bytes)
1541 print_length (expected_bytes, restval, false);
1542
1543 #ifdef HAVE_SSL
1544 if (u->scheme == SCHEME_FTPS && using_data_security)
1545 {
1546 /* We should try to restore the existing SSL session in the data connection
1547 * and fall back to establishing a new session if the server doesn't want to restore it.
1548 */
1549 if (!opt.ftps_resume_ssl || !ssl_connect_wget (dtsock, u->host, &csock))
1550 {
1551 if (opt.ftps_resume_ssl)
1552 logputs (LOG_NOTQUIET, "Server does not want to resume the SSL session. Trying with a new one.\n");
1553 if (!ssl_connect_wget (dtsock, u->host, NULL))
1554 {
1555 fd_close (csock);
1556 fd_close (dtsock);
1557 err = CONERROR;
1558 logputs (LOG_NOTQUIET, "Could not perform SSL handshake.\n");
1559 goto exit_error;
1560 }
1561 }
1562 else
1563 logputs (LOG_NOTQUIET, "Resuming SSL session in data connection.\n");
1564
1565 if (!ssl_check_certificate (dtsock, u->host))
1566 {
1567 fd_close (csock);
1568 fd_close (dtsock);
1569 err = CONERROR;
1570 goto exit_error;
1571 }
1572 }
1573 #endif
1574
1575 /* Get the contents of the document. */
1576 flags = 0;
1577 if (restval && rest_failed)
1578 flags |= rb_skip_startpos;
1579 rd_size = 0;
1580 res = fd_read_body (con->target, dtsock, fp,
1581 expected_bytes ? expected_bytes - restval : 0,
1582 restval, &rd_size, qtyread, &con->dltime, flags, warc_tmp);
1583
1584 tms = datetime_str (time (NULL));
1585 tmrate = retr_rate (rd_size, con->dltime);
1586 total_download_time += con->dltime;
1587
1588 #ifdef ENABLE_XATTR
1589 if (opt.enable_xattr)
1590 set_file_metadata (u, NULL, fp);
1591 #endif
1592
1593 fd_close (local_sock);
1594 /* Close the local file. */
1595 if (!output_stream || con->cmd & DO_LIST)
1596 fclose (fp);
1597
1598 /* If fd_read_body couldn't write to fp or warc_tmp, bail out. */
1599 if (res == -2 || (warc_tmp != NULL && res == -3))
1600 {
1601 logprintf (LOG_NOTQUIET, _("%s: %s, closing control connection.\n"),
1602 con->target, strerror (errno));
1603 fd_close (csock);
1604 con->csock = -1;
1605 fd_close (dtsock);
1606 if (res == -2)
1607 return FWRITEERR;
1608 else if (res == -3)
1609 return WARC_TMP_FWRITEERR;
1610 }
1611 else if (res == -1)
1612 {
1613 logprintf (LOG_NOTQUIET, _("%s (%s) - Data connection: %s; "),
1614 tms, tmrate, fd_errstr (dtsock));
1615 if (opt.server_response)
1616 logputs (LOG_ALWAYS, "\n");
1617 }
1618 fd_close (dtsock);
1619
1620 /* Get the server to tell us if everything is retrieved. */
1621 err = ftp_response (csock, &respline);
1622 if (err != FTPOK)
1623 {
1624 /* The control connection is decidedly closed. Print the time
1625 only if it hasn't already been printed. */
1626 if (res != -1)
1627 logprintf (LOG_NOTQUIET, "%s (%s) - ", tms, tmrate);
1628 logputs (LOG_NOTQUIET, _("Control connection closed.\n"));
1629 /* If there is an error on the control connection, close it, but
1630 return FTPRETRINT, since there is a possibility that the
1631 whole file was retrieved nevertheless (but that is for
1632 ftp_loop_internal to decide). */
1633 fd_close (csock);
1634 con->csock = -1;
1635 return FTPRETRINT;
1636 } /* err != FTPOK */
1637 *last_expected_bytes = ftp_expected_bytes (respline);
1638 /* If retrieval failed for any reason, return FTPRETRINT, but do not
1639 close socket, since the control connection is still alive. If
1640 there is something wrong with the control connection, it will
1641 become apparent later. */
1642 if (*respline != '2')
1643 {
1644 if (res != -1)
1645 logprintf (LOG_NOTQUIET, "%s (%s) - ", tms, tmrate);
1646 logputs (LOG_NOTQUIET, _("Data transfer aborted.\n"));
1647 #ifdef HAVE_SSL
1648 if (!c_strncasecmp (respline, "425", 3) && u->scheme == SCHEME_FTPS)
1649 {
1650 logputs (LOG_NOTQUIET, "FTPS server rejects new SSL sessions in the data connection.\n");
1651 xfree (respline);
1652 return FTPRESTFAIL;
1653 }
1654 #endif
1655 xfree (respline);
1656 return FTPRETRINT;
1657 }
1658 xfree (respline);
1659
1660 if (res == -1)
1661 {
1662 /* What now? The data connection was erroneous, whereas the
1663 response says everything is OK. We shall play it safe. */
1664 return FTPRETRINT;
1665 }
1666
1667 if (!(cmd & LEAVE_PENDING))
1668 {
1669 /* Closing the socket is faster than sending 'QUIT' and the
1670 effect is the same. */
1671 fd_close (csock);
1672 con->csock = -1;
1673 }
1674 /* If it was a listing, and opt.server_response is true,
1675 print it out. */
1676 if (con->cmd & DO_LIST)
1677 {
1678 if (opt.server_response)
1679 {
1680 /* 2005-02-25 SMS.
1681 Much of this work may already have been done, but repeating it should
1682 do no damage beyond wasting time.
1683 */
1684 /* On VMS, alter the name as required. */
1685 #ifdef __VMS
1686 char *targ;
1687
1688 targ = ods_conform (con->target);
1689 if (targ != con->target)
1690 {
1691 xfree (con->target);
1692 con->target = targ;
1693 }
1694 #endif /* def __VMS */
1695
1696 mkalldirs (con->target);
1697 fp = fopen (con->target, "r");
1698 if (!fp)
1699 logprintf (LOG_ALWAYS, "%s: %s\n", con->target, strerror (errno));
1700 else
1701 {
1702 char *line = NULL;
1703 size_t bufsize = 0;
1704 ssize_t len;
1705
1706 /* The lines are being read with getline because of
1707 no-buffering on opt.lfile. */
1708 while ((len = getline (&line, &bufsize, fp)) > 0)
1709 {
1710 while (len > 0 && (line[len - 1] == '\n' || line[len - 1] == '\r'))
1711 line[--len] = '\0';
1712 logprintf (LOG_ALWAYS, "%s\n",
1713 quotearg_style (escape_quoting_style, line));
1714 }
1715 xfree (line);
1716 fclose (fp);
1717 }
1718 } /* server_response */
1719
1720 /* 2013-10-17 Andrea Urbani (matfanjol)
1721 < __LIST_A_EXPLANATION__ >
1722 After the SYST command, looks if it knows that system.
1723 If yes, wget will force the use of "LIST" or "LIST -a".
1724 If no, wget will try, only the first time of each session, before the
1725 "LIST -a" command and after the "LIST".
1726 If "LIST -a" works and returns more or equal data of the "LIST",
1727 "LIST -a" will be the standard list command for all the session.
1728 If "LIST -a" fails or returns less data than "LIST" (think on the case
1729 of an existing file called "-a"), "LIST" will be the standard list
1730 command for all the session.
1731 ("LIST -a" is used to get also the hidden files)
1732
1733 */
1734 if (!(con->st & LIST_AFTER_LIST_A_CHECK_DONE))
1735 {
1736 /* We still have to check "LIST" after the first "LIST -a" to see
1737 if with "LIST" we get more data than "LIST -a", that means
1738 "LIST -a" returned files/folders with "-a" name. */
1739 if (con->st & AVOID_LIST_A)
1740 {
1741 /* LIST was used in this cycle.
1742 Let's see the result. */
1743 if (rd_size > previous_rd_size)
1744 {
1745 /* LIST returns more data than "LIST -a".
1746 "LIST" is the official command to use. */
1747 con->st |= LIST_AFTER_LIST_A_CHECK_DONE;
1748 DEBUGP (("LIST returned more data than \"LIST -a\": "
1749 "I will use \"LIST\" as standard list command\n"));
1750 }
1751 else if (previous_rd_size > rd_size)
1752 {
1753 /* "LIST -a" returned more data then LIST.
1754 "LIST -a" is the official command to use. */
1755 con->st |= LIST_AFTER_LIST_A_CHECK_DONE;
1756 con->st |= AVOID_LIST;
1757 con->st &= ~AVOID_LIST_A;
1758 /* Sorry, please, download again the "LIST -a"... */
1759 try_again = true;
1760 DEBUGP (("LIST returned less data than \"LIST -a\": I will "
1761 "use \"LIST -a\" as standard list command\n"));
1762 }
1763 else
1764 {
1765 /* LIST and "LIST -a" return the same data. */
1766 if (rd_size == 0)
1767 {
1768 /* Same empty data. We will check both again because
1769 we cannot check if "LIST -a" has returned an empty
1770 folder instead of a folder content. */
1771 con->st &= ~AVOID_LIST_A;
1772 }
1773 else
1774 {
1775 /* Same data, so, better to take "LIST -a" that
1776 shows also hidden files/folders (when present) */
1777 con->st |= LIST_AFTER_LIST_A_CHECK_DONE;
1778 con->st |= AVOID_LIST;
1779 con->st &= ~AVOID_LIST_A;
1780 DEBUGP (("LIST returned the same amount of data of "
1781 "\"LIST -a\": I will use \"LIST -a\" as standard "
1782 "list command\n"));
1783 }
1784 }
1785 }
1786 else
1787 {
1788 /* In this cycle "LIST -a" should being used. Is it true? */
1789 if (list_a_used)
1790 {
1791 /* Yes, it is.
1792 OK, let's save the amount of data and try again
1793 with LIST */
1794 previous_rd_size = rd_size;
1795 try_again = true;
1796 con->st |= AVOID_LIST_A;
1797 }
1798 else
1799 {
1800 /* No: something happens and LIST was used.
1801 This means "LIST -a" raises an error. */
1802 con->st |= LIST_AFTER_LIST_A_CHECK_DONE;
1803 con->st |= AVOID_LIST_A;
1804 DEBUGP (("\"LIST -a\" failed: I will use \"LIST\" "
1805 "as standard list command\n"));
1806 }
1807 }
1808 }
1809 }
1810 } while (try_again);
1811 return RETRFINISHED;
1812
1813 exit_error:
1814
1815 /* If fp is a regular file, close and try to remove it */
1816 if (fp && (!output_stream || con->cmd & DO_LIST))
1817 fclose (fp);
1818 return err;
1819 }
1820
1821 /* A one-file FTP loop. This is the part where FTP retrieval is
1822 retried, and retried, and retried, and...
1823
1824 This loop either gets commands from con, or (if ON_YOUR_OWN is
1825 set), makes them up to retrieve the file given by the URL. */
1826 static uerr_t
1827 ftp_loop_internal (struct url *u, struct url *original_url, struct fileinfo *f,
1828 ccon *con, char **local_file, bool force_full_retrieve)
1829 {
1830 int count, orig_lp;
1831 wgint restval, len = 0, qtyread = 0;
1832 char *tms, *locf;
1833 const char *tmrate = NULL;
1834 uerr_t err;
1835 struct stat st;
1836
1837 /* Declare WARC variables. */
1838 bool warc_enabled = (opt.warc_filename != NULL);
1839 FILE *warc_tmp = NULL;
1840 ip_address warc_ip_buf, *warc_ip = NULL;
1841 wgint last_expected_bytes = 0;
1842
1843 /* Get the target, and set the name for the message accordingly. */
1844 if ((f == NULL) && (con->target))
1845 {
1846 /* Explicit file (like ".listing"). */
1847 locf = con->target;
1848 }
1849 else
1850 {
1851 /* URL-derived file. Consider "-O file" name. */
1852 xfree (con->target);
1853 con->target = url_file_name (opt.trustservernames || !original_url ? u : original_url, NULL);
1854 if (!opt.output_document)
1855 locf = con->target;
1856 else
1857 locf = opt.output_document;
1858 }
1859
1860 /* If the output_document was given, then this check was already done and
1861 the file didn't exist. Hence the !opt.output_document */
1862
1863 /* If we receive .listing file it is necessary to determine system type of the ftp
1864 server even if opn.noclobber is given. Thus we must ignore opt.noclobber in
1865 order to establish connection with the server and get system type. */
1866 if (opt.noclobber && !opt.output_document && file_exists_p (con->target, NULL)
1867 && !((con->cmd & DO_LIST) && !(con->cmd & DO_RETR)))
1868 {
1869 logprintf (LOG_VERBOSE,
1870 _("File %s already there; not retrieving.\n"), quote (con->target));
1871 /* If the file is there, we suppose it's retrieved OK. */
1872 return RETROK;
1873 }
1874
1875 /* Remove it if it's a link. */
1876 remove_link (con->target);
1877
1878 count = 0;
1879
1880 if (con->st & ON_YOUR_OWN)
1881 con->st = ON_YOUR_OWN;
1882
1883 orig_lp = con->cmd & LEAVE_PENDING ? 1 : 0;
1884
1885 /* THE loop. */
1886 do
1887 {
1888 /* Increment the pass counter. */
1889 ++count;
1890 sleep_between_retrievals (count);
1891 if (con->st & ON_YOUR_OWN)
1892 {
1893 con->cmd = 0;
1894 con->cmd |= (DO_RETR | LEAVE_PENDING);
1895 if (con->csock != -1)
1896 con->cmd &= ~ (DO_LOGIN | DO_CWD);
1897 else
1898 con->cmd |= (DO_LOGIN | DO_CWD);
1899 }
1900 else /* not on your own */
1901 {
1902 if (con->csock != -1)
1903 con->cmd &= ~DO_LOGIN;
1904 else
1905 con->cmd |= DO_LOGIN;
1906 if (con->st & DONE_CWD)
1907 con->cmd &= ~DO_CWD;
1908 else
1909 con->cmd |= DO_CWD;
1910 }
1911
1912 /* For file RETR requests, we can write a WARC record.
1913 We record the file contents to a temporary file. */
1914 if (warc_enabled && (con->cmd & DO_RETR) && warc_tmp == NULL)
1915 {
1916 warc_tmp = warc_tempfile ();
1917 if (warc_tmp == NULL)
1918 return WARC_TMP_FOPENERR;
1919
1920 if (!con->proxy && con->csock != -1)
1921 {
1922 warc_ip = &warc_ip_buf;
1923 socket_ip_address (con->csock, warc_ip, ENDPOINT_PEER);
1924 }
1925 }
1926
1927 /* Decide whether or not to restart. */
1928 if (con->cmd & DO_LIST)
1929 restval = 0;
1930 else if (force_full_retrieve)
1931 restval = 0;
1932 else if (opt.start_pos >= 0)
1933 restval = opt.start_pos;
1934 else if (opt.always_rest
1935 && stat (locf, &st) == 0
1936 && S_ISREG (st.st_mode))
1937 /* When -c is used, continue from on-disk size. (Can't use
1938 hstat.len even if count>1 because we don't want a failed
1939 first attempt to clobber existing data.) */
1940 restval = st.st_size;
1941 else if (count > 1)
1942 restval = qtyread; /* start where the previous run left off */
1943 else
1944 restval = 0;
1945
1946 /* Get the current time string. */
1947 tms = datetime_str (time (NULL));
1948 /* Print fetch message, if opt.verbose. */
1949 if (opt.verbose)
1950 {
1951 char *hurl = url_string (u, URL_AUTH_HIDE_PASSWD);
1952 char tmp[256];
1953 strcpy (tmp, " ");
1954 if (count > 1)
1955 sprintf (tmp, _("(try:%2d)"), count);
1956 logprintf (LOG_VERBOSE, "--%s-- %s\n %s => %s\n",
1957 tms, hurl, tmp, quote (locf));
1958 #ifdef WINDOWS
1959 ws_changetitle (hurl);
1960 #endif
1961 xfree (hurl);
1962 }
1963 /* Send getftp the proper length, if fileinfo was provided. */
1964 if (f && f->type != FT_SYMLINK)
1965 len = f->size;
1966 else
1967 len = 0;
1968
1969 /* If we are working on a WARC record, getftp should also write
1970 to the warc_tmp file. */
1971 err = getftp (u, original_url, len, &qtyread, restval, con, count,
1972 &last_expected_bytes, warc_tmp);
1973
1974 if (con->csock == -1)
1975 con->st &= ~DONE_CWD;
1976 else
1977 con->st |= DONE_CWD;
1978
1979 switch (err)
1980 {
1981 case HOSTERR: case CONIMPOSSIBLE: case FWRITEERR: case FOPENERR:
1982 case FTPNSFOD: case FTPLOGINC: case FTPNOPASV: case FTPNOAUTH: case FTPNOPBSZ: case FTPNOPROT:
1983 case UNLINKERR: case WARC_TMP_FWRITEERR: case CONSSLERR: case CONTNOTSUPPORTED:
1984 case VERIFCERTERR:
1985 #ifdef HAVE_SSL
1986 if (err == FTPNOAUTH)
1987 logputs (LOG_NOTQUIET, "Server does not support AUTH TLS.\n");
1988 if (opt.ftps_implicit)
1989 logputs (LOG_NOTQUIET, "Server does not like implicit FTPS connections.\n");
1990 #endif
1991 /* Fatal errors, give up. */
1992 if (warc_tmp != NULL)
1993 {
1994 fclose (warc_tmp);
1995 warc_tmp = NULL;
1996 }
1997 return err;
1998 case CONSOCKERR: case CONERROR: case FTPSRVERR: case FTPRERR:
1999 case WRITEFAILED: case FTPUNKNOWNTYPE: case FTPSYSERR:
2000 case FTPPORTERR: case FTPLOGREFUSED: case FTPINVPASV:
2001 case FOPEN_EXCL_ERR:
2002 printwhat (count, opt.ntry);
2003 /* non-fatal errors */
2004 if (err == FOPEN_EXCL_ERR)
2005 {
2006 /* Re-determine the file name. */
2007 xfree (con->target);
2008 con->target = url_file_name (u, NULL);
2009 locf = con->target;
2010 }
2011 continue;
2012 case FTPRETRINT:
2013 /* If the control connection was closed, the retrieval
2014 will be considered OK if f->size == len. */
2015 if (!f || qtyread != f->size)
2016 {
2017 printwhat (count, opt.ntry);
2018 continue;
2019 }
2020 break;
2021 case RETRFINISHED:
2022 /* Great! */
2023 break;
2024 default:
2025 /* Not as great. */
2026 abort ();
2027 }
2028 tms = datetime_str (time (NULL));
2029 if (!opt.spider)
2030 tmrate = retr_rate (qtyread - restval, con->dltime);
2031
2032 /* If we get out of the switch above without continue'ing, we've
2033 successfully downloaded a file. Remember this fact. */
2034 downloaded_file (FILE_DOWNLOADED_NORMALLY, locf);
2035
2036 if (con->st & ON_YOUR_OWN)
2037 {
2038 fd_close (con->csock);
2039 con->csock = -1;
2040 }
2041 if (!opt.spider)
2042 {
2043 bool write_to_stdout = (opt.output_document && HYPHENP (opt.output_document));
2044
2045 logprintf (LOG_VERBOSE,
2046 write_to_stdout
2047 ? _("%s (%s) - written to stdout %s[%s]\n\n")
2048 : _("%s (%s) - %s saved [%s]\n\n"),
2049 tms, tmrate,
2050 write_to_stdout ? "" : quote (locf),
2051 number_to_static_string (qtyread));
2052 }
2053 if (!opt.verbose && !opt.quiet)
2054 {
2055 /* Need to hide the password from the URL. The `if' is here
2056 so that we don't do the needless allocation every
2057 time. */
2058 char *hurl = url_string (u, URL_AUTH_HIDE_PASSWD);
2059 logprintf (LOG_NONVERBOSE, "%s URL: %s [%s] -> \"%s\" [%d]\n",
2060 tms, hurl, number_to_static_string (qtyread), locf, count);
2061 xfree (hurl);
2062 }
2063
2064 if (warc_enabled && (con->cmd & DO_RETR))
2065 {
2066 /* Create and store a WARC resource record for the retrieved file. */
2067 bool warc_res;
2068
2069 warc_res = warc_write_resource_record (NULL, u->url, NULL, NULL,
2070 warc_ip, NULL, warc_tmp, -1);
2071
2072 if (! warc_res)
2073 return WARC_ERR;
2074
2075 /* warc_write_resource_record has also closed warc_tmp. */
2076 warc_tmp = NULL;
2077 }
2078
2079 if (con->cmd & DO_LIST)
2080 /* This is a directory listing file. */
2081 {
2082 if (!opt.remove_listing)
2083 /* --dont-remove-listing was specified, so do count this towards the
2084 number of bytes and files downloaded. */
2085 {
2086 total_downloaded_bytes += (qtyread - restval);
2087 numurls++;
2088 }
2089
2090 /* Deletion of listing files is not controlled by --delete-after, but
2091 by the more specific option --dont-remove-listing, and the code
2092 to do this deletion is in another function. */
2093 }
2094 else if (!opt.spider)
2095 /* This is not a directory listing file. */
2096 {
2097 /* Unlike directory listing files, don't pretend normal files weren't
2098 downloaded if they're going to be deleted. People seeding proxies,
2099 for instance, may want to know how many bytes and files they've
2100 downloaded through it. */
2101 total_downloaded_bytes += (qtyread - restval);
2102 numurls++;
2103
2104 if (opt.delete_after && !input_file_url (opt.input_filename))
2105 {
2106 DEBUGP (("\
2107 Removing file due to --delete-after in ftp_loop_internal():\n"));
2108 logprintf (LOG_VERBOSE, _("Removing %s.\n"), locf);
2109 if (unlink (locf))
2110 logprintf (LOG_NOTQUIET, "unlink: %s\n", strerror (errno));
2111 }
2112 }
2113
2114 /* Restore the original leave-pendingness. */
2115 if (orig_lp)
2116 con->cmd |= LEAVE_PENDING;
2117 else
2118 con->cmd &= ~LEAVE_PENDING;
2119
2120 if (local_file)
2121 *local_file = xstrdup (locf);
2122
2123 if (warc_tmp != NULL)
2124 {
2125 fclose (warc_tmp);
2126 warc_tmp = NULL;
2127 }
2128
2129 return RETROK;
2130 } while (!opt.ntry || (count < opt.ntry));
2131
2132 if (con->csock != -1 && (con->st & ON_YOUR_OWN))
2133 {
2134 fd_close (con->csock);
2135 con->csock = -1;
2136 }
2137
2138 if (warc_tmp != NULL)
2139 fclose (warc_tmp);
2140
2141 return TRYLIMEXC;
2142 }
2143
2144 /* Return the directory listing in a reusable format. The directory
2145 is specified in u->dir. */
2146 static uerr_t
2147 ftp_get_listing (struct url *u, struct url *original_url, ccon *con,
2148 struct fileinfo **f)
2149 {
2150 uerr_t err;
2151 char *uf; /* url file name */
2152 char *lf; /* list file name */
2153 char *old_target = con->target;
2154
2155 con->st &= ~ON_YOUR_OWN;
2156 con->cmd |= (DO_LIST | LEAVE_PENDING);
2157 con->cmd &= ~DO_RETR;
2158
2159 /* Find the listing file name. We do it by taking the file name of
2160 the URL and replacing the last component with the listing file
2161 name. */
2162 uf = url_file_name (u, NULL);
2163 lf = file_merge (uf, LIST_FILENAME);
2164 xfree (uf);
2165 DEBUGP ((_("Using %s as listing tmp file.\n"), quote (lf)));
2166
2167 con->target = xstrdup (lf);
2168 xfree (lf);
2169 err = ftp_loop_internal (u, original_url, NULL, con, NULL, false);
2170 lf = xstrdup (con->target);
2171 xfree (con->target);
2172 con->target = old_target;
2173
2174 if (err == RETROK)
2175 {
2176 *f = ftp_parse_ls (lf, con->rs);
2177 if (opt.remove_listing)
2178 {
2179 if (unlink (lf))
2180 logprintf (LOG_NOTQUIET, "unlink: %s\n", strerror (errno));
2181 else
2182 logprintf (LOG_VERBOSE, _("Removed %s.\n"), quote (lf));
2183 }
2184 }
2185 else
2186 *f = NULL;
2187 xfree (lf);
2188 con->cmd &= ~DO_LIST;
2189 return err;
2190 }
2191
2192 static uerr_t ftp_retrieve_dirs (struct url *, struct url *,
2193 struct fileinfo *, ccon *);
2194 static uerr_t ftp_retrieve_glob (struct url *, struct url *, ccon *, int);
2195 static struct fileinfo *delelement (struct fileinfo **, struct fileinfo **);
2196
2197 /* Retrieve a list of files given in struct fileinfo linked list. If
2198 a file is a symbolic link, do not retrieve it, but rather try to
2199 set up a similar link on the local disk, if the symlinks are
2200 supported.
2201
2202 If opt.recursive is set, after all files have been retrieved,
2203 ftp_retrieve_dirs will be called to retrieve the directories. */
2204 static uerr_t
2205 ftp_retrieve_list (struct url *u, struct url *original_url,
2206 struct fileinfo *f, ccon *con)
2207 {
2208 static int depth = 0;
2209 uerr_t err;
2210 struct fileinfo *orig;
2211 wgint local_size;
2212 time_t tml;
2213 bool dlthis; /* Download this (file). */
2214 const char *actual_target = NULL;
2215 bool force_full_retrieve = false;
2216
2217 /* Increase the depth. */
2218 ++depth;
2219 if (opt.reclevel != INFINITE_RECURSION && depth > opt.reclevel)
2220 {
2221 DEBUGP ((_("Recursion depth %d exceeded max. depth %d.\n"),
2222 depth, opt.reclevel));
2223 --depth;
2224 return RECLEVELEXC;
2225 }
2226
2227 assert (f != NULL);
2228 orig = f;
2229
2230 con->st &= ~ON_YOUR_OWN;
2231 if (!(con->st & DONE_CWD))
2232 con->cmd |= DO_CWD;
2233 else
2234 con->cmd &= ~DO_CWD;
2235 con->cmd |= (DO_RETR | LEAVE_PENDING);
2236
2237 if (con->csock < 0)
2238 con->cmd |= DO_LOGIN;
2239 else
2240 con->cmd &= ~DO_LOGIN;
2241
2242 err = RETROK; /* in case it's not used */
2243
2244 while (f)
2245 {
2246 char *old_target, *ofile;
2247
2248 if (opt.quota && total_downloaded_bytes > opt.quota)
2249 {
2250 --depth;
2251 return QUOTEXC;
2252 }
2253 old_target = con->target;
2254
2255 ofile = xstrdup (u->file);
2256 url_set_file (u, f->name);
2257
2258 con->target = url_file_name (u, NULL);
2259 err = RETROK;
2260
2261 dlthis = true;
2262 if (opt.timestamping && f->type == FT_PLAINFILE)
2263 {
2264 struct stat st;
2265 /* If conversion of HTML files retrieved via FTP is ever implemented,
2266 we'll need to stat() <file>.orig here when -K has been specified.
2267 I'm not implementing it now since files on an FTP server are much
2268 more likely than files on an HTTP server to legitimately have a
2269 .orig suffix. */
2270 if (!stat (con->target, &st))
2271 {
2272 bool eq_size;
2273 bool cor_val;
2274 /* Else, get it from the file. */
2275 local_size = st.st_size;
2276 tml = st.st_mtime;
2277 #ifdef WINDOWS
2278 /* Modification time granularity is 2 seconds for Windows, so
2279 increase local time by 1 second for later comparison. */
2280 tml++;
2281 #endif
2282 /* Compare file sizes only for servers that tell us correct
2283 values. Assume sizes being equal for servers that lie
2284 about file size. */
2285 cor_val = (con->rs == ST_UNIX || con->rs == ST_WINNT);
2286 eq_size = cor_val ? (local_size == f->size) : true;
2287 if (f->tstamp <= tml && eq_size)
2288 {
2289 /* Remote file is older, file sizes can be compared and
2290 are both equal. */
2291 logprintf (LOG_VERBOSE, _("\
2292 Remote file no newer than local file %s -- not retrieving.\n"), quote (con->target));
2293 dlthis = false;
2294 }
2295 else if (f->tstamp > tml)
2296 {
2297 /* Remote file is newer */
2298 force_full_retrieve = true;
2299 logprintf (LOG_VERBOSE, _("\
2300 Remote file is newer than local file %s -- retrieving.\n\n"),
2301 quote (con->target));
2302 }
2303 else
2304 {
2305 /* Sizes do not match */
2306 logprintf (LOG_VERBOSE, _("\
2307 The sizes do not match (local %s) -- retrieving.\n\n"),
2308 number_to_static_string (local_size));
2309 }
2310 }
2311 } /* opt.timestamping && f->type == FT_PLAINFILE */
2312 switch (f->type)
2313 {
2314 case FT_SYMLINK:
2315 /* If opt.retr_symlinks is defined, we treat symlinks as
2316 if they were normal files. There is currently no way
2317 to distinguish whether they might be directories, and
2318 follow them. */
2319 if (!opt.retr_symlinks)
2320 {
2321 #ifdef HAVE_SYMLINK
2322 if (!f->linkto)
2323 logputs (LOG_NOTQUIET,
2324 _("Invalid name of the symlink, skipping.\n"));
2325 else
2326 {
2327 struct stat st;
2328 /* Check whether we already have the correct
2329 symbolic link. */
2330 int rc = lstat (con->target, &st);
2331 if (rc == 0)
2332 {
2333 size_t len = strlen (f->linkto) + 1;
2334 if (S_ISLNK (st.st_mode))
2335 {
2336 char buf[1024], *link_target;
2337 size_t n;
2338 bool res;
2339
2340 if (len < sizeof (buf))
2341 link_target = buf;
2342 else
2343 link_target = xmalloc (len);
2344
2345 n = readlink (con->target, link_target, len);
2346 res = (n == len - 1) && (memcmp (link_target, f->linkto, n) == 0);
2347
2348 if (link_target != buf)
2349 xfree (link_target);
2350
2351 if (res)
2352 {
2353 logprintf (LOG_VERBOSE, _("\
2354 Already have correct symlink %s -> %s\n\n"),
2355 quote_n (0, con->target),
2356 quote_n (1, f->linkto));
2357 dlthis = false;
2358 break;
2359 }
2360 }
2361 }
2362 logprintf (LOG_VERBOSE, _("Creating symlink %s -> %s\n"),
2363 quote_n (0, con->target), quote_n (1, f->linkto));
2364 /* Unlink before creating symlink! */
2365 unlink (con->target);
2366 if (symlink (f->linkto, con->target) == -1)
2367 logprintf (LOG_NOTQUIET, "symlink: %s\n", strerror (errno));
2368 logputs (LOG_VERBOSE, "\n");
2369 } /* have f->linkto */
2370 #else /* not HAVE_SYMLINK */
2371 logprintf (LOG_NOTQUIET,
2372 _("Symlinks not supported, skipping symlink %s.\n"),
2373 quote (con->target));
2374 #endif /* not HAVE_SYMLINK */
2375 }
2376 else /* opt.retr_symlinks */
2377 {
2378 if (dlthis)
2379 {
2380 err = ftp_loop_internal (u, original_url, f, con, NULL,
2381 force_full_retrieve);
2382 }
2383 } /* opt.retr_symlinks */
2384 break;
2385 case FT_DIRECTORY:
2386 if (!opt.recursive)
2387 logprintf (LOG_NOTQUIET, _("Skipping directory %s.\n"),
2388 quote (f->name));
2389 break;
2390 case FT_PLAINFILE:
2391 /* Call the retrieve loop. */
2392 if (dlthis)
2393 {
2394 err = ftp_loop_internal (u, original_url, f, con, NULL,
2395 force_full_retrieve);
2396 }
2397 break;
2398 case FT_UNKNOWN:
2399 default:
2400 logprintf (LOG_NOTQUIET, _("%s: unknown/unsupported file type.\n"),
2401 quote (f->name));
2402 break;
2403 } /* switch */
2404
2405
2406 /* 2004-12-15 SMS.
2407 * Set permissions _before_ setting the times, as setting the
2408 * permissions changes the modified-time, at least on VMS.
2409 * Also, use the opt.output_document name here, too, as
2410 * appropriate. (Do the test once, and save the result.)
2411 */
2412
2413 set_local_file (&actual_target, con->target);
2414
2415 /* If downloading a plain file, and the user requested it, then
2416 set valid (non-zero) permissions. */
2417 if (dlthis && (actual_target != NULL) &&
2418 (f->type == FT_PLAINFILE) && opt.preserve_perm)
2419 {
2420 if (f->perms)
2421 {
2422 if (chmod (actual_target, f->perms))
2423 logprintf (LOG_NOTQUIET,
2424 _("Failed to set permissions for %s.\n"),
2425 actual_target);
2426 }
2427 else
2428 DEBUGP (("Unrecognized permissions for %s.\n", actual_target));
2429 }
2430
2431 /* Set the time-stamp information to the local file. Symlinks
2432 are not to be stamped because it sets the stamp on the
2433 original. :( */
2434 if (actual_target != NULL)
2435 {
2436 if (opt.useservertimestamps
2437 && !(f->type == FT_SYMLINK && !opt.retr_symlinks)
2438 && f->tstamp != -1
2439 && dlthis
2440 && file_exists_p (con->target, NULL))
2441 {
2442 touch (actual_target, f->tstamp);
2443 }
2444 else if (f->tstamp == -1)
2445 logprintf (LOG_NOTQUIET, _("%s: corrupt time-stamp.\n"),
2446 actual_target);
2447 }
2448
2449 xfree (con->target);
2450 con->target = old_target;
2451
2452 url_set_file (u, ofile);
2453 xfree (ofile);
2454
2455 /* Break on fatals. */
2456 if (err == QUOTEXC || err == HOSTERR || err == FWRITEERR
2457 || err == WARC_ERR || err == WARC_TMP_FOPENERR
2458 || err == WARC_TMP_FWRITEERR)
2459 break;
2460 con->cmd &= ~ (DO_CWD | DO_LOGIN);
2461 f = f->next;
2462 }
2463
2464 /* We do not want to call ftp_retrieve_dirs here */
2465 if (opt.recursive &&
2466 !(opt.reclevel != INFINITE_RECURSION && depth >= opt.reclevel))
2467 err = ftp_retrieve_dirs (u, original_url, orig, con);
2468 else if (opt.recursive)
2469 DEBUGP ((_("Will not retrieve dirs since depth is %d (max %d).\n"),
2470 depth, opt.reclevel));
2471 --depth;
2472 return err;
2473 }
2474
2475 /* Retrieve the directories given in a file list. This function works
2476 by simply going through the linked list and calling
2477 ftp_retrieve_glob on each directory entry. The function knows
2478 about excluded directories. */
2479 static uerr_t
2480 ftp_retrieve_dirs (struct url *u, struct url *original_url,
2481 struct fileinfo *f, ccon *con)
2482 {
2483 char buf[1024];
2484 char *container = buf;
2485 int container_size = sizeof (buf);
2486
2487 for (; f; f = f->next)
2488 {
2489 int size;
2490 char *odir, *newdir;
2491
2492 if (opt.quota && total_downloaded_bytes > opt.quota)
2493 break;
2494 if (f->type != FT_DIRECTORY)
2495 continue;
2496
2497 /* Allocate u->dir off stack, but reallocate only if a larger
2498 string is needed. It's a pity there's no "realloca" for an
2499 item on the bottom of the stack. */
2500 size = strlen (u->dir) + 1 + strlen (f->name) + 1;
2501 if (size > container_size)
2502 {
2503 if (container == buf)
2504 container = xmalloc (size);
2505 else
2506 container = xrealloc (container, size);
2507
2508 container_size = size;
2509 }
2510 newdir = container;
2511
2512 odir = u->dir;
2513 if (*odir == '\0'
2514 || (*odir == '/' && *(odir + 1) == '\0'))
2515 /* If ODIR is empty or just "/", simply append f->name to
2516 ODIR. (In the former case, to preserve u->dir being
2517 relative; in the latter case, to avoid double slash.) */
2518 sprintf (newdir, "%s%s", odir, f->name);
2519 else
2520 /* Else, use a separator. */
2521 sprintf (newdir, "%s/%s", odir, f->name);
2522
2523 DEBUGP (("Composing new CWD relative to the initial directory.\n"));
2524 DEBUGP ((" odir = '%s'\n f->name = '%s'\n newdir = '%s'\n\n",
2525 odir, f->name, newdir));
2526 if (!accdir (newdir))
2527 {
2528 logprintf (LOG_VERBOSE, _("\
2529 Not descending to %s as it is excluded/not-included.\n"),
2530 quote (newdir));
2531 continue;
2532 }
2533
2534 con->st &= ~DONE_CWD;
2535
2536 odir = xstrdup (u->dir); /* because url_set_dir will free
2537 u->dir. */
2538 url_set_dir (u, newdir);
2539 ftp_retrieve_glob (u, original_url, con, GLOB_GETALL);
2540 url_set_dir (u, odir);
2541 xfree (odir);
2542
2543 /* Set the time-stamp? */
2544 }
2545
2546 if (container != buf)
2547 xfree (container);
2548
2549 if (opt.quota && total_downloaded_bytes > opt.quota)
2550 return QUOTEXC;
2551 else
2552 return RETROK;
2553 }
2554
2555 /* Return true if S has a leading '/' or contains '../' */
2556 static bool
2557 has_insecure_name_p (const char *s)
2558 {
2559 if (*s == '/')
2560 return true;
2561
2562 if (strstr (s, "../") != 0)
2563 return true;
2564
2565 return false;
2566 }
2567
2568 /* Test if the file node is invalid. This can occur due to malformed or
2569 * maliciously crafted listing files being returned by the server.
2570 *
2571 * Currently, this function only tests if there are multiple entries in the
2572 * listing file by the same name. However this function can be expanded as more
2573 * such illegal listing formats are discovered. */
2574 static bool
2575 is_invalid_entry (struct fileinfo *f)
2576 {
2577 struct fileinfo *cur = f;
2578 char *f_name = f->name;
2579
2580 /* If the node we're currently checking has a duplicate later, we eliminate
2581 * the current node and leave the next one intact. */
2582 while (cur->next)
2583 {
2584 cur = cur->next;
2585 if (strcmp (f_name, cur->name) == 0)
2586 return true;
2587 }
2588 return false;
2589 }
2590
2591 /* A near-top-level function to retrieve the files in a directory.
2592 The function calls ftp_get_listing, to get a linked list of files.
2593 Then it weeds out the file names that do not match the pattern.
2594 ftp_retrieve_list is called with this updated list as an argument.
2595
2596 If the argument ACTION is GLOB_GETONE, just download the file (but
2597 first get the listing, so that the time-stamp is heeded); if it's
2598 GLOB_GLOBALL, use globbing; if it's GLOB_GETALL, download the whole
2599 directory. */
2600 static uerr_t
2601 ftp_retrieve_glob (struct url *u, struct url *original_url,
2602 ccon *con, int action)
2603 {
2604 struct fileinfo *f, *start;
2605 uerr_t res;
2606
2607 con->cmd |= LEAVE_PENDING;
2608
2609 res = ftp_get_listing (u, original_url, con, &start);
2610 if (res != RETROK)
2611 return res;
2612
2613 // Set the function used for glob matching.
2614 int (*matcher) (const char *, const char *, int)
2615 = opt.ignore_case ? fnmatch_nocase : fnmatch;
2616
2617 // Set the function used to compare strings
2618 #ifdef __VMS
2619 /* 2009-09-09 SMS.
2620 * Odd-ball compiler ("HP C V7.3-009 on OpenVMS Alpha V7.3-2")
2621 * bug causes spurious %CC-E-BADCONDIT complaint with this
2622 * "?:" statement. (Different linkage attributes for strcmp()
2623 * and strcasecmp().) Converting to "if" changes the
2624 * complaint to %CC-W-PTRMISMATCH on "cmp = strcmp;". Adding
2625 * the senseless type cast clears the complaint, and looks
2626 * harmless.
2627 */
2628 int (*cmp) (const char *, const char *)
2629 = opt.ignore_case ? strcasecmp : (int (*)())strcmp;
2630 #else /* def __VMS */
2631 int (*cmp) (const char *, const char *)
2632 = opt.ignore_case ? strcasecmp : strcmp;
2633 #endif /* def __VMS [else] */
2634
2635 f = start;
2636 while (f)
2637 {
2638
2639 // Weed out files that do not confirm to the global rules given in
2640 // opt.accepts and opt.rejects
2641 if ((opt.accepts || opt.rejects) &&
2642 f->type != FT_DIRECTORY && !acceptable (f->name))
2643 {
2644 logprintf (LOG_VERBOSE, _("Rejecting %s.\n"),
2645 quote (f->name));
2646 f = delelement (&f, &start);
2647 continue;
2648 }
2649
2650
2651 // Identify and eliminate possibly harmful names or invalid entries.
2652 if (has_insecure_name_p (f->name) || is_invalid_entry (f))
2653 {
2654 logprintf (LOG_VERBOSE, _("Rejecting %s (Invalid Entry).\n"),
2655 quote (f->name));
2656 f = delelement (&f, &start);
2657 continue;
2658 }
2659
2660 if (opt.acceptregex || opt.rejectregex)
2661 {
2662 // accept_url() takes the full URL.
2663 char buf[1024];
2664 char *url = buf;
2665
2666 if ((unsigned) snprintf(buf, sizeof(buf), "%s%s%s",
2667 u->url, f->name, f->type == FT_DIRECTORY ? "/" : "")
2668 >= sizeof(buf))
2669 {
2670 url = aprintf("%s%s%s", u->url, f->name, f->type == FT_DIRECTORY ? "/" : "");
2671 }
2672
2673 if (!accept_url (url))
2674 {
2675 logprintf (LOG_VERBOSE, _ ("%s is excluded/not-included through regex.\n"), url);
2676 f = delelement (&f, &start);
2677 if (url != buf)
2678 xfree(url);
2679 continue;
2680 }
2681
2682 if (url != buf)
2683 xfree(url);
2684 }
2685
2686 /* Now weed out the files that do not match our globbing pattern.
2687 If we are dealing with a globbing pattern, that is. */
2688 if (*u->file)
2689 {
2690 if (action == GLOB_GLOBALL)
2691 {
2692 int matchres = matcher (u->file, f->name, 0);
2693 if (matchres == -1)
2694 {
2695 logprintf (LOG_NOTQUIET, _("Error matching %s against %s: %s\n"),
2696 u->file, quotearg_style (escape_quoting_style, f->name),
2697 strerror (errno));
2698 freefileinfo (start);
2699 return RETRBADPATTERN;
2700 }
2701 if (matchres == FNM_NOMATCH)
2702 {
2703 f = delelement (&f, &start); /* delete the element from the list */
2704 continue;
2705 }
2706 }
2707 else if (action == GLOB_GETONE)
2708 {
2709 if (0 != cmp(u->file, f->name))
2710 {
2711 f = delelement (&f, &start);
2712 continue;
2713 }
2714 }
2715 }
2716 f = f->next;
2717 }
2718
2719 /*
2720 * Now that preprocessing of the file listing is over, let's try to download
2721 * all the remaining files in our listing.
2722 */
2723 if (start)
2724 {
2725 /* Just get everything. */
2726 res = ftp_retrieve_list (u, original_url, start, con);
2727 }
2728 else
2729 {
2730 if (action == GLOB_GLOBALL)
2731 {
2732 /* No luck. */
2733 /* #### This message SUCKS. We should see what was the
2734 reason that nothing was retrieved. */
2735 logprintf (LOG_VERBOSE, _("No matches on pattern %s.\n"),
2736 quote (u->file));
2737 }
2738 else if (action == GLOB_GETONE) /* GLOB_GETONE or GLOB_GETALL */
2739 {
2740 /* Let's try retrieving it anyway. */
2741 con->st |= ON_YOUR_OWN;
2742 res = ftp_loop_internal (u, original_url, NULL, con, NULL, false);
2743 return res;
2744 }
2745
2746 /* If action == GLOB_GETALL, and the file list is empty, there's
2747 no point in trying to download anything or in complaining about
2748 it. (An empty directory should not cause complaints.)
2749 */
2750 }
2751 freefileinfo (start);
2752 if (opt.quota && total_downloaded_bytes > opt.quota)
2753 return QUOTEXC;
2754 else
2755 return res;
2756 }
2757
2758 /* The wrapper that calls an appropriate routine according to contents
2759 of URL. Inherently, its capabilities are limited on what can be
2760 encoded into a URL. */
2761 uerr_t
2762 ftp_loop (struct url *u, struct url *original_url, char **local_file, int *dt,
2763 struct url *proxy, bool recursive, bool glob)
2764 {
2765 ccon con; /* FTP connection */
2766 uerr_t res;
2767
2768 *dt = 0;
2769
2770 xzero (con);
2771
2772 con.csock = -1;
2773 con.st = ON_YOUR_OWN;
2774 con.rs = ST_UNIX;
2775 con.id = NULL;
2776 con.proxy = proxy;
2777
2778 /* If the file name is empty, the user probably wants a directory
2779 index. We'll provide one, properly HTML-ized. Unless
2780 opt.htmlify is 0, of course. :-) */
2781 if (!*u->file && !recursive)
2782 {
2783 struct fileinfo *f;
2784 res = ftp_get_listing (u, original_url, &con, &f);
2785
2786 if (res == RETROK)
2787 {
2788 if (opt.htmlify && !opt.spider)
2789 {
2790 struct url *url_file = opt.trustservernames ? u : original_url;
2791 char *filename = (opt.output_document
2792 ? xstrdup (opt.output_document)
2793 : (con.target ? xstrdup (con.target)
2794 : url_file_name (url_file, NULL)));
2795 res = ftp_index (filename, u, f);
2796 if (res == FTPOK && opt.verbose)
2797 {
2798 if (!opt.output_document)
2799 {
2800 struct stat st;
2801 wgint sz;
2802 if (stat (filename, &st) == 0)
2803 sz = st.st_size;
2804 else
2805 sz = -1;
2806 logprintf (LOG_NOTQUIET,
2807 _("Wrote HTML-ized index to %s [%s].\n"),
2808 quote (filename), number_to_static_string (sz));
2809 }
2810 else
2811 logprintf (LOG_NOTQUIET,
2812 _("Wrote HTML-ized index to %s.\n"),
2813 quote (filename));
2814 }
2815 xfree (filename);
2816 }
2817 freefileinfo (f);
2818 }
2819 }
2820 else
2821 {
2822 bool ispattern = false;
2823 if (glob)
2824 {
2825 /* Treat the URL as a pattern if the file name part of the
2826 URL path contains wildcards. (Don't check for u->file
2827 because it is unescaped and therefore doesn't leave users
2828 the option to escape literal '*' as %2A.) */
2829 char *file_part = strrchr (u->path, '/');
2830 if (!file_part)
2831 file_part = u->path;
2832 ispattern = has_wildcards_p (file_part);
2833 }
2834 if (ispattern || recursive || opt.timestamping || opt.preserve_perm)
2835 {
2836 /* ftp_retrieve_glob is a catch-all function that gets called
2837 if we need globbing, time-stamping, recursion or preserve
2838 permissions. Its third argument is just what we really need. */
2839 res = ftp_retrieve_glob (u, original_url, &con,
2840 ispattern ? GLOB_GLOBALL : GLOB_GETONE);
2841 }
2842 else
2843 {
2844 res = ftp_loop_internal (u, original_url, NULL, &con, local_file, false);
2845 }
2846 }
2847 if (res == FTPOK)
2848 res = RETROK;
2849 if (res == RETROK)
2850 *dt |= RETROKF;
2851 /* If a connection was left, quench it. */
2852 if (con.csock != -1)
2853 fd_close (con.csock);
2854 xfree (con.id);
2855 xfree (con.target);
2856 return res;
2857 }
2858
2859 /* Delete an element from the fileinfo linked list. Returns the
2860 address of the next element, or NULL if the list is exhausted. It
2861 can modify the start of the list. */
2862 static struct fileinfo *
2863 delelement (struct fileinfo **f, struct fileinfo **start)
2864 {
2865 struct fileinfo *prev = (*f)->prev;
2866 struct fileinfo *next = (*f)->next;
2867
2868 xfree ((*f)->name);
2869 xfree ((*f)->linkto);
2870 xfree (*f);
2871
2872 if (next)
2873 next->prev = prev;
2874 if (prev)
2875 prev->next = next;
2876 else
2877 *start = next;
2878 return next;
2879 }
2880
2881 /* Free the fileinfo linked list of files. */
2882 void
2883 freefileinfo (struct fileinfo *f)
2884 {
2885 while (f)
2886 {
2887 struct fileinfo *next = f->next;
2888 xfree (f->name);
2889 xfree (f->linkto);
2890 xfree (f);
2891 f = next;
2892 }
2893 }