Debian Codegraph @

src/retr.c

raw
1 /* File retrieval.
2 Copyright (C) 1996-2011, 2014-2015, 2018-2024 Free Software
3 Foundation, Inc.
4
5 This file is part of GNU Wget.
6
7 GNU Wget is free software; you can redistribute it and/or modify
8 it under the terms of the GNU General Public License as published by
9 the Free Software Foundation; either version 3 of the License, or (at
10 your option) any later version.
11
12 GNU Wget is distributed in the hope that it will be useful,
13 but WITHOUT ANY WARRANTY; without even the implied warranty of
14 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 GNU General Public License for more details.
16
17 You should have received a copy of the GNU General Public License
18 along with Wget. If not, see <http://www.gnu.org/licenses/>.
19
20 Additional permission under GNU GPL version 3 section 7
21
22 If you modify this program, or any covered work, by linking or
23 combining it with the OpenSSL project's OpenSSL library (or a
24 modified version of that library), containing parts covered by the
25 terms of the OpenSSL or SSLeay licenses, the Free Software Foundation
26 grants you additional permission to convey the resulting work.
27 Corresponding Source for a non-source form of such a combination
28 shall include the source code for the parts of OpenSSL used as well
29 as that of the covered work. */
30
31 #include "wget.h"
32
33 #include <stdio.h>
34 #include <stdlib.h>
35 #include <unistd.h>
36 #include <errno.h>
37 #include <string.h>
38 #include <assert.h>
39 #ifdef VMS
40 # include <unixio.h> /* For delete(). */
41 #endif
42
43 #ifdef HAVE_LIBZ
44 # include <zlib.h>
45 #endif
46
47 #ifdef HAVE_LIBPROXY
48 # include "proxy.h"
49 #endif
50
51 #include "exits.h"
52 #include "utils.h"
53 #include "retr.h"
54 #include "progress.h"
55 #include "url.h"
56 #include "recur.h"
57 #include "ftp.h"
58 #include "http.h"
59 #include "host.h"
60 #include "connect.h"
61 #include "convert.h"
62 #include "ptimer.h"
63 #include "html-url.h"
64 #include "iri.h"
65 #include "hsts.h"
66
67 /* Total size of downloaded files. Used to enforce quota. */
68 wgint total_downloaded_bytes;
69
70 /* Total download time in seconds. */
71 double total_download_time;
72
73 /* If non-NULL, the stream to which output should be written. This
74 stream is initialized when `-O' is used. */
75 FILE *output_stream;
76
77 /* Whether output_document is a regular file we can manipulate,
78 i.e. not `-' or a device file. */
79 bool output_stream_regular;
80
81 static struct {
82 wgint chunk_bytes;
83 double chunk_start;
84 double sleep_adjust;
85 } limit_data;
86
87 static void
88 limit_bandwidth_reset (void)
89 {
90 xzero (limit_data);
91 }
92
93 #ifdef HAVE_LIBZ
94 static voidpf
95 zalloc (voidpf opaque, unsigned int items, unsigned int size)
96 {
97 (void) opaque;
98 return (voidpf) xcalloc (items, size);
99 }
100
101 static void
102 zfree (voidpf opaque, voidpf address)
103 {
104 (void) opaque;
105 xfree (address);
106 }
107 #endif
108
109 /* Limit the bandwidth by pausing the download for an amount of time.
110 BYTES is the number of bytes received from the network, and TIMER
111 is the timer that started at the beginning of download. */
112
113 static void
114 limit_bandwidth (wgint bytes, struct ptimer *timer)
115 {
116 double delta_t = ptimer_read (timer) - limit_data.chunk_start;
117 double expected;
118
119 limit_data.chunk_bytes += bytes;
120
121 /* Calculate the amount of time we expect downloading the chunk
122 should take. If in reality it took less time, sleep to
123 compensate for the difference. */
124 expected = (double) limit_data.chunk_bytes / opt.limit_rate;
125
126 if (expected > delta_t)
127 {
128 double slp = expected - delta_t + limit_data.sleep_adjust;
129 double t0, t1;
130 if (slp < 0.2)
131 {
132 DEBUGP (("deferring a %.2f ms sleep (%s/%.2f).\n",
133 slp * 1000, number_to_static_string (limit_data.chunk_bytes),
134 delta_t));
135 return;
136 }
137 DEBUGP (("\nsleeping %.2f ms for %s bytes, adjust %.2f ms\n",
138 slp * 1000, number_to_static_string (limit_data.chunk_bytes),
139 limit_data.sleep_adjust));
140
141 t0 = ptimer_read (timer);
142 xsleep (slp);
143 t1 = ptimer_measure (timer);
144
145 /* Due to scheduling, we probably slept slightly longer (or
146 shorter) than desired. Calculate the difference between the
147 desired and the actual sleep, and adjust the next sleep by
148 that amount. */
149 limit_data.sleep_adjust = slp - (t1 - t0);
150 /* If sleep_adjust is very large, it's likely due to suspension
151 and not clock inaccuracy. Don't enforce those. */
152 if (limit_data.sleep_adjust > 0.5)
153 limit_data.sleep_adjust = 0.5;
154 else if (limit_data.sleep_adjust < -0.5)
155 limit_data.sleep_adjust = -0.5;
156 }
157
158 limit_data.chunk_bytes = 0;
159 limit_data.chunk_start = ptimer_read (timer);
160 }
161
162 /* Write data in BUF to OUT. However, if *SKIP is non-zero, skip that
163 amount of data and decrease SKIP. Increment *TOTAL by the amount
164 of data written. If OUT2 is not NULL, also write BUF to OUT2.
165 In case of error writing to OUT, -2 is returned. In case of error
166 writing to OUT2, -3 is returned. Return 1 if the whole BUF was
167 skipped. */
168
169 static int
170 write_data (FILE *out, FILE *out2, const char *buf, int bufsize,
171 wgint *skip, wgint *written)
172 {
173 if (out == NULL && out2 == NULL)
174 return 1;
175
176 if (skip)
177 {
178 if (*skip > bufsize)
179 {
180 *skip -= bufsize;
181 return 1;
182 }
183 if (*skip)
184 {
185 buf += *skip;
186 bufsize -= *skip;
187 *skip = 0;
188 if (bufsize == 0)
189 return 1;
190 }
191 }
192
193 if (out)
194 fwrite (buf, 1, bufsize, out);
195 if (out2)
196 fwrite (buf, 1, bufsize, out2);
197
198 if (written)
199 *written += bufsize;
200
201 /* Immediately flush the downloaded data. This should not hinder
202 performance: fast downloads will arrive in large 16K chunks
203 (which stdio would write out immediately anyway), and slow
204 downloads wouldn't be limited by disk speed. */
205
206 /* 2005-04-20 SMS.
207 Perhaps it shouldn't hinder performance, but it sure does, at least
208 on VMS (more than 2X). Rather than speculate on what it should or
209 shouldn't do, it might make more sense to test it. Even better, it
210 might be nice to explain what possible benefit it could offer, as
211 it appears to be a clear invitation to poor performance with no
212 actual justification. (Also, why 16K? Anyone test other values?)
213 */
214 #ifndef __VMS
215 if (out)
216 fflush (out);
217 if (out2)
218 fflush (out2);
219 #endif /* ndef __VMS */
220
221 if (out && ferror (out))
222 return -2;
223 else if (out2 && ferror (out2))
224 return -3;
225
226 return 0;
227 }
228
229 /* Read the contents of file descriptor FD until it the connection
230 terminates or a read error occurs. The data is read in portions of
231 up to 16K and written to OUT as it arrives. If opt.verbose is set,
232 the progress is shown.
233
234 TOREAD is the amount of data expected to arrive, normally only used
235 by the progress gauge.
236
237 STARTPOS is the position from which the download starts, used by
238 the progress gauge. If QTYREAD is non-NULL, the value it points to
239 is incremented by the amount of data read from the network. If
240 QTYWRITTEN is non-NULL, the value it points to is incremented by
241 the amount of data written to disk. The time it took to download
242 the data is stored to ELAPSED.
243
244 If OUT2 is non-NULL, the contents is also written to OUT2.
245 OUT2 will get an exact copy of the response: if this is a chunked
246 response, everything -- including the chunk headers -- is written
247 to OUT2. (OUT will only get the unchunked response.)
248
249 The function exits and returns the amount of data read. In case of
250 error while reading data, -1 is returned. In case of error while
251 writing data to OUT, -2 is returned. In case of error while writing
252 data to OUT2, -3 is returned. */
253
254 int
255 fd_read_body (const char *downloaded_filename, int fd, FILE *out, wgint toread, wgint startpos,
256
257 wgint *qtyread, wgint *qtywritten, double *elapsed, int flags,
258 FILE *out2)
259 {
260 int ret = 0;
261 int dlbufsize = MAX (BUFSIZ, 64 * 1024);
262 char *dlbuf = xmalloc (dlbufsize);
263
264 struct ptimer *timer = NULL;
265 double last_successful_read_tm = 0;
266
267 /* The progress gauge, set according to the user preferences. */
268 void *progress = NULL;
269
270 /* Non-zero if the progress gauge is interactive, i.e. if it can
271 continually update the display. When true, smaller timeout
272 values are used so that the gauge can update the display when
273 data arrives slowly. */
274 bool progress_interactive = false;
275
276 bool exact = !!(flags & rb_read_exactly);
277
278 /* Used only by HTTP/HTTPS chunked transfer encoding. */
279 bool chunked = flags & rb_chunked_transfer_encoding;
280 wgint skip = 0;
281
282 /* How much data we've read/written. */
283 wgint sum_read = 0;
284 wgint sum_written = 0;
285 wgint remaining_chunk_size = 0;
286
287 #ifdef HAVE_LIBZ
288 /* try to minimize the number of calls to inflate() and write_data() per
289 call to fd_read() */
290 unsigned int gzbufsize = dlbufsize * 4;
291 char *gzbuf = NULL;
292 z_stream gzstream;
293
294 if (flags & rb_compressed_gzip)
295 {
296 gzbuf = xmalloc (gzbufsize);
297 gzstream.zalloc = zalloc;
298 gzstream.zfree = zfree;
299 gzstream.opaque = Z_NULL;
300 gzstream.next_in = Z_NULL;
301 gzstream.avail_in = 0;
302
303 #define GZIP_DETECT 32 /* gzip format detection */
304 #define GZIP_WINDOW 15 /* logarithmic window size (default: 15) */
305 ret = inflateInit2 (&gzstream, GZIP_DETECT | GZIP_WINDOW);
306 if (ret != Z_OK)
307 {
308 xfree (gzbuf);
309 errno = (ret == Z_MEM_ERROR) ? ENOMEM : EINVAL;
310 ret = -1;
311 goto out;
312 }
313 }
314 #endif
315
316 if (flags & rb_skip_startpos)
317 skip = startpos;
318
319 if (opt.show_progress)
320 {
321 const char *filename_progress;
322 /* If we're skipping STARTPOS bytes, pass 0 as the INITIAL
323 argument to progress_create because the indicator doesn't
324 (yet) know about "skipping" data. */
325 wgint start = skip ? 0 : startpos;
326 if (opt.dir_prefix)
327 filename_progress = downloaded_filename + strlen (opt.dir_prefix) + 1;
328 else
329 filename_progress = downloaded_filename;
330 progress = progress_create (filename_progress, start, start + toread);
331 progress_interactive = progress_interactive_p (progress);
332 }
333
334 if (opt.limit_rate)
335 limit_bandwidth_reset ();
336
337 /* A timer is needed for tracking progress, for throttling, and for
338 tracking elapsed time. If either of these are requested, start
339 the timer. */
340 if (progress || opt.limit_rate || elapsed)
341 {
342 timer = ptimer_new ();
343 last_successful_read_tm = 0;
344 }
345
346 /* Use a smaller buffer for low requested bandwidths. For example,
347 with --limit-rate=2k, it doesn't make sense to slurp in 16K of
348 data and then sleep for 8s. With buffer size equal to the limit,
349 we never have to sleep for more than one second. */
350 if (opt.limit_rate && opt.limit_rate < dlbufsize)
351 dlbufsize = opt.limit_rate;
352
353 /* Read from FD while there is data to read. Normally toread==0
354 means that it is unknown how much data is to arrive. However, if
355 EXACT is set, then toread==0 means what it says: that no data
356 should be read. */
357 while (!exact || (sum_read < toread))
358 {
359 int rdsize;
360 double tmout = opt.read_timeout;
361
362 if (chunked)
363 {
364 if (remaining_chunk_size == 0)
365 {
366 char *line = fd_read_line (fd);
367 char *endl;
368 if (line == NULL)
369 {
370 ret = -1;
371 break;
372 }
373 else if (out2 != NULL)
374 fwrite (line, 1, strlen (line), out2);
375
376 remaining_chunk_size = strtol (line, &endl, 16);
377 xfree (line);
378
379 if (remaining_chunk_size < 0)
380 {
381 ret = -1;
382 break;
383 }
384
385 if (remaining_chunk_size == 0)
386 {
387 ret = 0;
388 line = fd_read_line (fd);
389 if (line == NULL)
390 ret = -1;
391 else
392 {
393 if (out2 != NULL)
394 fwrite (line, 1, strlen (line), out2);
395 xfree (line);
396 }
397 break;
398 }
399 }
400
401 rdsize = MIN (remaining_chunk_size, dlbufsize);
402 }
403 else
404 rdsize = exact ? MIN (toread - sum_read, dlbufsize) : dlbufsize;
405
406 if (progress_interactive)
407 {
408 /* For interactive progress gauges, always specify a ~1s
409 timeout, so that the gauge can be updated regularly even
410 when the data arrives very slowly or stalls. */
411 tmout = 0.95;
412 /* avoid wrong 'interactive timeout' */
413 errno = 0;
414 if (opt.read_timeout)
415 {
416 double waittm;
417 waittm = ptimer_read (timer) - last_successful_read_tm;
418 if (waittm + tmout > opt.read_timeout)
419 {
420 /* Don't let total idle time exceed read timeout. */
421 tmout = opt.read_timeout - waittm;
422 /* if 0 fd_read can be 'blocked read' */
423 if (tmout <= 0)
424 {
425 /* We've already exceeded the timeout. */
426 ret = -1, errno = ETIMEDOUT;
427 break;
428 }
429 }
430 }
431 }
432 ret = fd_read (fd, dlbuf, rdsize, tmout);
433
434 if (progress_interactive && ret < 0 && errno == ETIMEDOUT)
435 ret = 0; /* interactive timeout, handled above */
436 else if (ret <= 0)
437 break; /* EOF or read error */
438
439 if (progress || opt.limit_rate || elapsed)
440 {
441 ptimer_measure (timer);
442 if (ret > 0)
443 last_successful_read_tm = ptimer_read (timer);
444 }
445
446 if (ret > 0)
447 {
448 int write_res;
449
450 sum_read += ret;
451
452 #ifdef HAVE_LIBZ
453 if (gzbuf)
454 {
455 int err;
456 int towrite;
457
458 /* Write original data to WARC file */
459 write_res = write_data (NULL, out2, dlbuf, ret, NULL, NULL);
460 if (write_res < 0)
461 {
462 ret = write_res;
463 goto out;
464 }
465
466 gzstream.avail_in = ret;
467 gzstream.next_in = (unsigned char *) dlbuf;
468
469 do
470 {
471 gzstream.avail_out = gzbufsize;
472 gzstream.next_out = (unsigned char *) gzbuf;
473
474 err = inflate (&gzstream, Z_NO_FLUSH);
475
476 switch (err)
477 {
478 case Z_MEM_ERROR:
479 errno = ENOMEM;
480 ret = -1;
481 goto out;
482 case Z_NEED_DICT:
483 case Z_DATA_ERROR:
484 errno = EINVAL;
485 ret = -1;
486 goto out;
487 case Z_STREAM_END:
488 if (exact && sum_read != toread)
489 {
490 DEBUGP(("zlib stream ended unexpectedly after %"PRId64"/%"PRId64
491 " bytes\n", sum_read, toread));
492 }
493 }
494
495 towrite = gzbufsize - gzstream.avail_out;
496 write_res = write_data (out, NULL, gzbuf, towrite, &skip,
497 &sum_written);
498 if (write_res < 0)
499 {
500 ret = write_res;
501 goto out;
502 }
503 }
504 while (gzstream.avail_out == 0);
505 }
506 else
507 #endif
508 {
509 write_res = write_data (out, out2, dlbuf, ret, &skip,
510 &sum_written);
511 if (write_res < 0)
512 {
513 ret = write_res;
514 goto out;
515 }
516 }
517
518 if (chunked)
519 {
520 remaining_chunk_size -= ret;
521 if (remaining_chunk_size == 0)
522 {
523 char *line = fd_read_line (fd);
524 if (line == NULL)
525 {
526 ret = -1;
527 break;
528 }
529 else
530 {
531 if (out2 != NULL)
532 fwrite (line, 1, strlen (line), out2);
533 xfree (line);
534 }
535 }
536 }
537 }
538
539 if (opt.limit_rate)
540 limit_bandwidth (ret, timer);
541
542 if (progress)
543 progress_update (progress, ret, ptimer_read (timer));
544 #ifdef WINDOWS
545 if (toread > 0 && opt.show_progress)
546 ws_percenttitle (100.0 *
547 (startpos + sum_read) / (startpos + toread));
548 #endif
549 }
550 if (ret < -1)
551 ret = -1;
552
553 out:
554 if (progress)
555 progress_finish (progress, ptimer_read (timer));
556
557 if (timer)
558 {
559 if (elapsed)
560 *elapsed = ptimer_read (timer);
561 ptimer_destroy (timer);
562 }
563
564 #ifdef HAVE_LIBZ
565 if (gzbuf)
566 {
567 int err = inflateEnd (&gzstream);
568 if (ret >= 0)
569 {
570 /* with compression enabled, ret must be 0 if successful */
571 if (err == Z_OK)
572 ret = 0;
573 else
574 {
575 errno = EINVAL;
576 ret = -1;
577 }
578 }
579 xfree (gzbuf);
580
581 if (gzstream.total_in != (uLong) sum_read)
582 {
583 DEBUGP(("zlib read size differs from raw read size (%lu/%"PRId64")\n",
584 gzstream.total_in, sum_read));
585 }
586 }
587 #endif
588
589 if (qtyread)
590 *qtyread += sum_read;
591 if (qtywritten)
592 *qtywritten += sum_written;
593
594 xfree (dlbuf);
595
596 return ret;
597 }
598
599 /* Read a hunk of data from FD, up until a terminator. The hunk is
600 limited by whatever the TERMINATOR callback chooses as its
601 terminator. For example, if terminator stops at newline, the hunk
602 will consist of a line of data; if terminator stops at two
603 newlines, it can be used to read the head of an HTTP response.
604 Upon determining the boundary, the function returns the data (up to
605 the terminator) in malloc-allocated storage.
606
607 In case of read error, NULL is returned. In case of EOF and no
608 data read, NULL is returned and errno set to 0. In case of having
609 read some data, but encountering EOF before seeing the terminator,
610 the data that has been read is returned, but it will (obviously)
611 not contain the terminator.
612
613 The TERMINATOR function is called with three arguments: the
614 beginning of the data read so far, the beginning of the current
615 block of peeked-at data, and the length of the current block.
616 Depending on its needs, the function is free to choose whether to
617 analyze all data or just the newly arrived data. If TERMINATOR
618 returns NULL, it means that the terminator has not been seen.
619 Otherwise it should return a pointer to the charactre immediately
620 following the terminator.
621
622 The idea is to be able to read a line of input, or otherwise a hunk
623 of text, such as the head of an HTTP request, without crossing the
624 boundary, so that the next call to fd_read etc. reads the data
625 after the hunk. To achieve that, this function does the following:
626
627 1. Peek at incoming data.
628
629 2. Determine whether the peeked data, along with the previously
630 read data, includes the terminator.
631
632 2a. If yes, read the data until the end of the terminator, and
633 exit.
634
635 2b. If no, read the peeked data and goto 1.
636
637 The function is careful to assume as little as possible about the
638 implementation of peeking. For example, every peek is followed by
639 a read. If the read returns a different amount of data, the
640 process is retried until all data arrives safely.
641
642 SIZEHINT is the buffer size sufficient to hold all the data in the
643 typical case (it is used as the initial buffer size). MAXSIZE is
644 the maximum amount of memory this function is allowed to allocate,
645 or 0 if no upper limit is to be enforced.
646
647 This function should be used as a building block for other
648 functions -- see fd_read_line as a simple example. */
649
650 char *
651 fd_read_hunk (int fd, hunk_terminator_t terminator, long sizehint, long maxsize)
652 {
653 long bufsize = sizehint;
654 char *hunk = xmalloc (bufsize);
655 int tail = 0; /* tail position in HUNK */
656
657 assert (!maxsize || maxsize >= bufsize);
658
659 while (1)
660 {
661 const char *end;
662 int pklen, rdlen, remain;
663
664 /* First, peek at the available data. */
665
666 pklen = fd_peek (fd, hunk + tail, bufsize - 1 - tail, -1);
667 if (pklen < 0)
668 {
669 xfree (hunk);
670 return NULL;
671 }
672 end = terminator (hunk, hunk + tail, pklen);
673 if (end)
674 {
675 /* The data contains the terminator: we'll drain the data up
676 to the end of the terminator. */
677 remain = end - (hunk + tail);
678 assert (remain >= 0);
679 if (remain == 0)
680 {
681 /* No more data needs to be read. */
682 hunk[tail] = '\0';
683 return hunk;
684 }
685 if (bufsize - 1 < tail + remain)
686 {
687 bufsize = tail + remain + 1;
688 hunk = xrealloc (hunk, bufsize);
689 }
690 }
691 else
692 /* No terminator: simply read the data we know is (or should
693 be) available. */
694 remain = pklen;
695
696 /* Now, read the data. Note that we make no assumptions about
697 how much data we'll get. (Some TCP stacks are notorious for
698 read returning less data than the previous MSG_PEEK.) */
699
700 rdlen = fd_read (fd, hunk + tail, remain, 0);
701 if (rdlen < 0)
702 {
703 xfree (hunk);
704 return NULL;
705 }
706 tail += rdlen;
707 hunk[tail] = '\0';
708
709 if (rdlen == 0)
710 {
711 if (tail == 0)
712 {
713 /* EOF without anything having been read */
714 xfree (hunk);
715 errno = 0;
716 return NULL;
717 }
718 else
719 /* EOF seen: return the data we've read. */
720 return hunk;
721 }
722 if (end && rdlen == remain)
723 /* The terminator was seen and the remaining data drained --
724 we got what we came for. */
725 return hunk;
726
727 /* Keep looping until all the data arrives. */
728
729 if (tail == bufsize - 1)
730 {
731 /* Double the buffer size, but refuse to allocate more than
732 MAXSIZE bytes. */
733 if (maxsize && bufsize >= maxsize)
734 {
735 xfree (hunk);
736 errno = ENOMEM;
737 return NULL;
738 }
739 bufsize <<= 1;
740 if (maxsize && bufsize > maxsize)
741 bufsize = maxsize;
742 hunk = xrealloc (hunk, bufsize);
743 }
744 }
745 }
746
747 static const char *
748 line_terminator (const char *start _GL_UNUSED, const char *peeked, int peeklen)
749 {
750 const char *p = memchr (peeked, '\n', peeklen);
751 if (p)
752 /* p+1 because the line must include '\n' */
753 return p + 1;
754 return NULL;
755 }
756
757 /* The maximum size of the single line we agree to accept. This is
758 not meant to impose an arbitrary limit, but to protect the user
759 from Wget slurping up available memory upon encountering malicious
760 or buggy server output. Define it to 0 to remove the limit. */
761 #define FD_READ_LINE_MAX 4096
762
763 /* Read one line from FD and return it. The line is allocated using
764 malloc, but is never larger than FD_READ_LINE_MAX.
765
766 If an error occurs, or if no data can be read, NULL is returned.
767 In the former case errno indicates the error condition, and in the
768 latter case, errno is NULL. */
769
770 char *
771 fd_read_line (int fd)
772 {
773 return fd_read_hunk (fd, line_terminator, 128, FD_READ_LINE_MAX);
774 }
775
776 /* Return a printed representation of the download rate, along with
777 the units appropriate for the download speed. */
778
779 const char *
780 retr_rate (wgint bytes, double secs)
781 {
782 static char res[20];
783 static const char *rate_names[] = {"B/s", "KB/s", "MB/s", "GB/s", "TB/s" };
784 static const char *rate_names_bits[] = {"b/s", "Kb/s", "Mb/s", "Gb/s", "Tb/s" };
785 int units;
786
787 double dlrate = calc_rate (bytes, secs, &units);
788 /* Use more digits for smaller numbers (regardless of unit used),
789 e.g. "1022", "247", "12.5", "2.38". */
790 snprintf (res, sizeof(res), "%.*f %s",
791 dlrate >= 99.95 ? 0 : dlrate >= 9.995 ? 1 : 2,
792 dlrate, !opt.report_bps ? rate_names[units]: rate_names_bits[units]);
793
794 return res;
795 }
796
797 /* Calculate the download rate and trim it as appropriate for the
798 speed. Appropriate means that if rate is greater than 1K/s,
799 kilobytes are used, and if rate is greater than 1MB/s, megabytes
800 are used.
801
802 UNITS is zero for B/s, one for KB/s, two for MB/s, and three for
803 GB/s. */
804
805 double
806 calc_rate (wgint bytes, double secs, int *units)
807 {
808 double dlrate;
809 double bibyte;
810
811 if (!opt.report_bps)
812 bibyte = 1024.0;
813 else
814 bibyte = 1000.0;
815
816 if (secs == 0)
817 /* If elapsed time is exactly zero, it means we're under the
818 resolution of the timer. This can easily happen on systems
819 that use time() for the timer. Since the interval lies between
820 0 and the timer's resolution, assume half the resolution. */
821 secs = ptimer_resolution () / 2.0;
822
823 dlrate = secs ? convert_to_bits (bytes) / secs : 0;
824 if (dlrate < bibyte)
825 *units = 0;
826 else if (dlrate < (bibyte * bibyte))
827 *units = 1, dlrate /= bibyte;
828 else if (dlrate < (bibyte * bibyte * bibyte))
829 *units = 2, dlrate /= (bibyte * bibyte);
830 else if (dlrate < (bibyte * bibyte * bibyte * bibyte))
831 *units = 3, dlrate /= (bibyte * bibyte * bibyte);
832 else {
833 *units = 4, dlrate /= (bibyte * bibyte * bibyte * bibyte);
834 if (dlrate > 99.99)
835 dlrate = 99.99; // upper limit 99.99TB/s
836 }
837
838 return dlrate;
839 }
840
841
842 #define SUSPEND_METHOD do { \
843 method_suspended = true; \
844 saved_body_data = opt.body_data; \
845 saved_body_file_name = opt.body_file; \
846 saved_method = opt.method; \
847 opt.body_data = NULL; \
848 opt.body_file = NULL; \
849 opt.method = NULL; \
850 } while (0)
851
852 #define RESTORE_METHOD do { \
853 if (method_suspended) \
854 { \
855 opt.body_data = saved_body_data; \
856 opt.body_file = saved_body_file_name; \
857 opt.method = saved_method; \
858 method_suspended = false; \
859 } \
860 } while (0)
861
862 static char *getproxy (struct url *);
863
864 /* Retrieve the given URL. Decides which loop to call -- HTTP, FTP,
865 FTP, proxy, etc. */
866
867 /* #### This function should be rewritten so it doesn't return from
868 multiple points. */
869
870 uerr_t
871 retrieve_url (struct url * orig_parsed, const char *origurl, char **file,
872 char **newloc, const char *refurl, int *dt, bool recursive,
873 struct iri *iri, bool register_status)
874 {
875 uerr_t result;
876 char *url;
877 bool location_changed;
878 bool iri_fallbacked = 0;
879 int dummy;
880 char *mynewloc, *proxy;
881 struct url *u = orig_parsed, *proxy_url;
882 int up_error_code; /* url parse error code */
883 char *local_file = NULL;
884 int redirection_count = 0;
885
886 bool method_suspended = false;
887 char *saved_body_data = NULL;
888 char *saved_method = NULL;
889 char *saved_body_file_name = NULL;
890
891 /* If dt is NULL, use local storage. */
892 if (!dt)
893 {
894 dt = &dummy;
895 dummy = 0;
896 }
897 url = xstrdup (origurl);
898 if (newloc)
899 *newloc = NULL;
900 if (file)
901 *file = NULL;
902
903 if (!refurl)
904 refurl = opt.referer;
905
906 redirected:
907 /* (also for IRI fallbacking) */
908
909 result = NOCONERROR;
910 mynewloc = NULL;
911 xfree(local_file);
912 proxy_url = NULL;
913
914 proxy = getproxy (u);
915 if (proxy)
916 {
917 struct iri *pi = iri_new ();
918 set_uri_encoding (pi, opt.locale, true);
919 pi->utf8_encode = false;
920
921 /* Parse the proxy URL. */
922 proxy_url = url_parse (proxy, &up_error_code, pi, true);
923 if (!proxy_url)
924 {
925 logprintf (LOG_NOTQUIET, _("Error parsing proxy URL %s: %s.\n"),
926 proxy, url_error (up_error_code));
927 xfree (url);
928 xfree (proxy);
929 iri_free (pi);
930 RESTORE_METHOD;
931 result = PROXERR;
932 if (orig_parsed != u)
933 url_free (u);
934 goto bail;
935 }
936 if (proxy_url->scheme != SCHEME_HTTP && proxy_url->scheme != u->scheme)
937 {
938 logprintf (LOG_NOTQUIET, _("Error in proxy URL %s: Must be HTTP.\n"), proxy);
939 url_free (proxy_url);
940 xfree (url);
941 xfree (proxy);
942 iri_free (pi);
943 RESTORE_METHOD;
944 result = PROXERR;
945 if (orig_parsed != u)
946 url_free (u);
947 goto bail;
948 }
949 iri_free(pi);
950 xfree (proxy);
951 }
952
953 if (u->scheme == SCHEME_HTTP
954 #ifdef HAVE_SSL
955 || u->scheme == SCHEME_HTTPS
956 #endif
957 || (proxy_url && proxy_url->scheme == SCHEME_HTTP))
958 {
959 #ifdef HAVE_HSTS
960 #ifdef TESTING
961 /* we don't link against main.o when we're testing */
962 hsts_store_t hsts_store = NULL;
963 #else
964 extern hsts_store_t hsts_store;
965 #endif
966
967 if (opt.hsts && hsts_store)
968 {
969 if (hsts_match (hsts_store, u))
970 logprintf (LOG_VERBOSE, "URL transformed to HTTPS due to an HSTS policy\n");
971 }
972 #endif
973 result = http_loop (u, orig_parsed, &mynewloc, &local_file, refurl, dt,
974 proxy_url, iri);
975 }
976 else if (u->scheme == SCHEME_FTP
977 #ifdef HAVE_SSL
978 || u->scheme == SCHEME_FTPS
979 #endif
980 )
981 {
982 /* If this is a redirection, temporarily turn off opt.ftp_glob
983 and opt.recursive, both being undesirable when following
984 redirects. */
985 bool oldrec = recursive, glob = opt.ftp_glob;
986 if (redirection_count)
987 oldrec = glob = false;
988
989 result = ftp_loop (u, orig_parsed, &local_file, dt, proxy_url,
990 recursive, glob);
991 recursive = oldrec;
992
993 /* There is a possibility of having HTTP being redirected to
994 FTP. In these cases we must decide whether the text is HTML
995 according to the suffix. The HTML suffixes are `.html',
996 `.htm' and a few others, case-insensitive. */
997 if (redirection_count && local_file && (u->scheme == SCHEME_FTP
998 #ifdef HAVE_SSL
999 || u->scheme == SCHEME_FTPS
1000 #endif
1001 ))
1002 {
1003 if (has_html_suffix_p (local_file))
1004 *dt |= TEXTHTML;
1005 }
1006 }
1007
1008 if (proxy_url)
1009 {
1010 url_free (proxy_url);
1011 proxy_url = NULL;
1012 }
1013
1014 location_changed = (result == NEWLOCATION || result == NEWLOCATION_KEEP_POST);
1015 if (location_changed)
1016 {
1017 char *construced_newloc;
1018 struct url *newloc_parsed;
1019
1020 assert (mynewloc != NULL);
1021
1022 xfree (local_file);
1023
1024 /* The HTTP specs only allow absolute URLs to appear in
1025 redirects, but a ton of boneheaded webservers and CGIs out
1026 there break the rules and use relative URLs, and popular
1027 browsers are lenient about this, so wget should be too. */
1028 construced_newloc = uri_merge (url, mynewloc ? mynewloc : "");
1029 xfree (mynewloc);
1030 mynewloc = construced_newloc;
1031
1032 #ifdef ENABLE_IRI
1033 /* Reset UTF-8 encoding state, set the URI encoding and reset
1034 the content encoding. */
1035 iri->utf8_encode = opt.enable_iri;
1036 if (opt.encoding_remote)
1037 set_uri_encoding (iri, opt.encoding_remote, true);
1038 set_content_encoding (iri, NULL);
1039 xfree (iri->orig_url);
1040 #endif
1041
1042 /* Now, see if this new location makes sense. */
1043 newloc_parsed = url_parse (mynewloc, &up_error_code, iri, true);
1044 if (!newloc_parsed)
1045 {
1046 logprintf (LOG_NOTQUIET, "%s: %s.\n", escnonprint_uri (mynewloc),
1047 url_error (up_error_code));
1048 if (orig_parsed != u)
1049 {
1050 url_free (u);
1051 }
1052 xfree (url);
1053 xfree (mynewloc);
1054 RESTORE_METHOD;
1055 goto bail;
1056 }
1057
1058 /* Now mynewloc will become newloc_parsed->url, because if the
1059 Location contained relative paths like .././something, we
1060 don't want that propagating as url. */
1061 xfree (mynewloc);
1062 mynewloc = xstrdup (newloc_parsed->url);
1063
1064 /* Check for max. number of redirections. */
1065 if (++redirection_count > opt.max_redirect)
1066 {
1067 logprintf (LOG_NOTQUIET, _("%d redirections exceeded.\n"),
1068 opt.max_redirect);
1069 url_free (newloc_parsed);
1070 if (orig_parsed != u)
1071 {
1072 url_free (u);
1073 }
1074 xfree (url);
1075 xfree (mynewloc);
1076 RESTORE_METHOD;
1077 result = WRONGCODE;
1078 goto bail;
1079 }
1080
1081 xfree (url);
1082 url = mynewloc;
1083 if (orig_parsed != u)
1084 {
1085 url_free (u);
1086 }
1087 u = newloc_parsed;
1088
1089 /* If we're being redirected from POST, and we received a
1090 redirect code different than 307, we don't want to POST
1091 again. Many requests answer POST with a redirection to an
1092 index page; that redirection is clearly a GET. We "suspend"
1093 POST data for the duration of the redirections, and restore
1094 it when we're done.
1095
1096 RFC2616 HTTP/1.1 introduces code 307 Temporary Redirect
1097 specifically to preserve the method of the request.
1098 */
1099 if (result != NEWLOCATION_KEEP_POST && !method_suspended)
1100 SUSPEND_METHOD;
1101
1102 goto redirected;
1103 }
1104 else
1105 {
1106 xfree(mynewloc);
1107 }
1108
1109 /* Try to not encode in UTF-8 if fetching failed */
1110 if (!(*dt & RETROKF) && iri->utf8_encode)
1111 {
1112 iri->utf8_encode = false;
1113 if (orig_parsed != u)
1114 {
1115 url_free (u);
1116 }
1117 u = url_parse (origurl, NULL, iri, true);
1118 if (u)
1119 {
1120 if (strcmp(u->url, orig_parsed->url))
1121 {
1122 DEBUGP (("[IRI fallbacking to non-utf8 for %s\n", quote (url)));
1123 xfree (url);
1124 url = xstrdup (u->url);
1125 iri_fallbacked = 1;
1126 goto redirected;
1127 }
1128 else
1129 DEBUGP (("[Needn't fallback to non-utf8 for %s\n", quote (url)));
1130 }
1131 else
1132 DEBUGP (("[Couldn't fallback to non-utf8 for %s\n", quote (url)));
1133 }
1134
1135 if (local_file && u && (*dt & RETROKF || opt.content_on_error))
1136 {
1137 register_download (u->url, local_file);
1138
1139 if (!opt.spider && redirection_count && 0 != strcmp (origurl, u->url))
1140 register_redirection (origurl, u->url);
1141
1142 if (*dt & TEXTHTML)
1143 register_html (local_file);
1144
1145 if (*dt & TEXTCSS)
1146 register_css (local_file);
1147 }
1148
1149 if (file)
1150 *file = local_file ? local_file : NULL;
1151 else
1152 xfree (local_file);
1153
1154 if (orig_parsed != u)
1155 url_free (u);
1156
1157 if (redirection_count || iri_fallbacked)
1158 {
1159 if (newloc)
1160 *newloc = url;
1161 else
1162 xfree (url);
1163 }
1164 else
1165 {
1166 if (newloc)
1167 *newloc = NULL;
1168 xfree (url);
1169 }
1170
1171 RESTORE_METHOD;
1172
1173 bail:
1174 if (register_status)
1175 inform_exit_status (result);
1176
1177 return result;
1178 }
1179
1180 static uerr_t retrieve_from_url_list(struct urlpos *url_list, int *count, struct iri *iri)
1181 {
1182 struct urlpos *cur_url;
1183 uerr_t status;
1184
1185 status = RETROK; /* Suppose everything is OK. */
1186
1187 for (cur_url = url_list; cur_url; cur_url = cur_url->next, ++*count)
1188 {
1189 char *filename = NULL, *new_file = NULL, *proxy;
1190 int dt = 0;
1191 struct iri *tmpiri;
1192 struct url *parsed_url;
1193
1194 if (cur_url->ignore_when_downloading)
1195 continue;
1196
1197 if (opt.quota && total_downloaded_bytes > opt.quota)
1198 {
1199 status = QUOTEXC;
1200 break;
1201 }
1202
1203 tmpiri = iri_dup (iri);
1204 parsed_url = url_parse (cur_url->url->url, NULL, tmpiri, true);
1205
1206 proxy = getproxy (cur_url->url);
1207 if ((opt.recursive || opt.page_requisites)
1208 && ((cur_url->url->scheme != SCHEME_FTP
1209 #ifdef HAVE_SSL
1210 && cur_url->url->scheme != SCHEME_FTPS
1211 #endif
1212 ) || proxy))
1213 {
1214 int old_follow_ftp = opt.follow_ftp;
1215
1216 /* Turn opt.follow_ftp on in case of recursive FTP retrieval */
1217 if (cur_url->url->scheme == SCHEME_FTP
1218 #ifdef HAVE_SSL
1219 || cur_url->url->scheme == SCHEME_FTPS
1220 #endif
1221 )
1222 opt.follow_ftp = 1;
1223
1224 status = retrieve_tree (parsed_url ? parsed_url : cur_url->url,
1225 tmpiri);
1226
1227 opt.follow_ftp = old_follow_ftp;
1228 }
1229 else
1230 status = retrieve_url (parsed_url ? parsed_url : cur_url->url,
1231 cur_url->url->url, &filename,
1232 &new_file, NULL, &dt, opt.recursive, tmpiri,
1233 true);
1234 xfree (proxy);
1235
1236 if (parsed_url)
1237 url_free (parsed_url);
1238
1239 if (filename && opt.delete_after && file_exists_p (filename, NULL))
1240 {
1241 DEBUGP (("\
1242 Removing file due to --delete-after in retrieve_from_file():\n"));
1243 logprintf (LOG_VERBOSE, _("Removing %s.\n"), filename);
1244 if (unlink (filename))
1245 logprintf (LOG_NOTQUIET, "Failed to unlink %s: (%d) %s\n", filename, errno, strerror (errno));
1246 dt &= ~RETROKF;
1247 }
1248
1249 xfree (new_file);
1250 xfree (filename);
1251 iri_free (tmpiri);
1252 }
1253 return status;
1254 }
1255
1256 /* Find the URLs in the file and call retrieve_url() for each of them.
1257 If HTML is true, treat the file as HTML, and construct the URLs
1258 accordingly.
1259
1260 If opt.recursive is set, call retrieve_tree() for each file. */
1261
1262 uerr_t
1263 retrieve_from_file (const char *file, bool html, int *count)
1264 {
1265 uerr_t status;
1266 struct urlpos *url_list, *cur_url;
1267 struct iri *iri = iri_new();
1268
1269 char *input_file, *url_file = NULL;
1270 const char *url = file;
1271
1272 *count = 0; /* Reset the URL count. */
1273
1274 /* sXXXav : Assume filename and links in the file are in the locale */
1275 set_uri_encoding (iri, opt.locale, true);
1276 set_content_encoding (iri, opt.locale);
1277
1278 if (url_valid_scheme (url))
1279 {
1280 int dt,url_err;
1281 struct url *url_parsed = url_parse (url, &url_err, iri, true);
1282 if (!url_parsed)
1283 {
1284 logprintf (LOG_NOTQUIET, "%s: %s.\n", url, url_error (url_err));
1285 iri_free (iri);
1286 return URLERROR;
1287 }
1288
1289 if (!opt.base_href)
1290 opt.base_href = xstrdup (url);
1291
1292 status = retrieve_url (url_parsed, url, &url_file, NULL, NULL, &dt,
1293 false, iri, true);
1294 url_free (url_parsed);
1295
1296 if (!url_file || (status != RETROK))
1297 {
1298 iri_free (iri);
1299 return status;
1300 }
1301
1302 if (dt & TEXTHTML)
1303 html = true;
1304
1305 #ifdef ENABLE_IRI
1306 /* If we have a found a content encoding, use it.
1307 * ( == is okay, because we're checking for identical object) */
1308 if (iri->content_encoding != opt.locale)
1309 set_uri_encoding (iri, iri->content_encoding, false);
1310 #endif
1311
1312 /* Reset UTF-8 encode status */
1313 iri->utf8_encode = opt.enable_iri;
1314 xfree (iri->orig_url);
1315
1316 input_file = url_file;
1317 }
1318 else
1319 input_file = (char *) file;
1320
1321 bool read_again = false;
1322 do {
1323 url_list = (html ? get_urls_html (input_file, NULL, NULL, iri)
1324 : get_urls_file (input_file, &read_again));
1325
1326 status = retrieve_from_url_list(url_list, count, iri);
1327 } while (read_again);
1328
1329 xfree (url_file);
1330
1331 /* Free the linked list of URL-s. */
1332 free_urlpos (url_list);
1333
1334 iri_free (iri);
1335
1336 return status;
1337 }
1338
1339 /* Print `giving up', or `retrying', depending on the impending
1340 action. N1 and N2 are the attempt number and the attempt limit. */
1341 void
1342 printwhat (int n1, int n2)
1343 {
1344 logputs (LOG_VERBOSE, (n1 == n2) ? _("Giving up.\n\n") : _("Retrying.\n\n"));
1345 }
1346
1347 /* If opt.wait or opt.waitretry are specified, and if certain
1348 conditions are met, sleep the appropriate number of seconds. See
1349 the documentation of --wait and --waitretry for more information.
1350
1351 COUNT is the count of current retrieval, beginning with 1. */
1352
1353 void
1354 sleep_between_retrievals (int count)
1355 {
1356 static bool first_retrieval = true;
1357
1358 if (first_retrieval)
1359 {
1360 /* Don't sleep before the very first retrieval. */
1361 first_retrieval = false;
1362 return;
1363 }
1364
1365 if (opt.waitretry && count > 1)
1366 {
1367 /* If opt.waitretry is specified and this is a retry, wait for
1368 COUNT-1 number of seconds, or for opt.waitretry seconds. */
1369 if (count <= opt.waitretry)
1370 xsleep (count - 1);
1371 else
1372 xsleep (opt.waitretry);
1373 }
1374 else if (opt.wait)
1375 {
1376 if (!opt.random_wait || count > 1)
1377 /* If random-wait is not specified, or if we are sleeping
1378 between retries of the same download, sleep the fixed
1379 interval. */
1380 xsleep (opt.wait);
1381 else
1382 {
1383 /* Sleep a random amount of time averaging in opt.wait
1384 seconds. The sleeping amount ranges from 0.5*opt.wait to
1385 1.5*opt.wait. */
1386 double waitsecs = (0.5 + random_float ()) * opt.wait;
1387 DEBUGP (("sleep_between_retrievals: avg=%f,sleep=%f\n",
1388 opt.wait, waitsecs));
1389 xsleep (waitsecs);
1390 }
1391 }
1392 }
1393
1394 /* Free the linked list of urlpos. */
1395 void
1396 free_urlpos (struct urlpos *l)
1397 {
1398 while (l)
1399 {
1400 struct urlpos *next = l->next;
1401 if (l->url)
1402 url_free (l->url);
1403 xfree (l->local_name);
1404 xfree (l);
1405 l = next;
1406 }
1407 }
1408
1409 /* Rotate FNAME opt.backups times */
1410 void
1411 rotate_backups(const char *fname)
1412 {
1413 #ifdef __VMS
1414 # define SEP "_"
1415 # define AVS ";*" /* All-version suffix. */
1416 # define AVSL (sizeof (AVS) - 1)
1417 #else
1418 # define SEP "."
1419 # define AVSL 0
1420 #endif
1421 #define FILE_BUF_SIZE 1024
1422
1423 /* avoid alloca() here */
1424 char from[FILE_BUF_SIZE], to[FILE_BUF_SIZE];
1425 struct stat sb;
1426 bool overflow;
1427 int i;
1428
1429 if (stat (fname, &sb) == 0)
1430 if (S_ISREG (sb.st_mode) == 0)
1431 return;
1432
1433 for (i = opt.backups; i > 1; i--)
1434 {
1435 #ifdef VMS
1436 /* Delete (all versions of) any existing max-suffix file, to avoid
1437 * creating multiple versions of it. (On VMS, rename() will
1438 * create a new version of an existing destination file, not
1439 * destroy/overwrite it.)
1440 */
1441 if (i == opt.backups)
1442 {
1443 if (((unsigned) snprintf (to, sizeof (to), "%s%s%d%s", fname, SEP, i, AVS)) >= sizeof (to))
1444 logprintf (LOG_NOTQUIET, "Failed to delete %s: File name truncation\n", to);
1445 else
1446 delete (to);
1447 }
1448 #endif
1449 overflow = (unsigned) snprintf (to, FILE_BUF_SIZE, "%s%s%d", fname, SEP, i) >= FILE_BUF_SIZE;
1450 overflow |= (unsigned) snprintf (from, FILE_BUF_SIZE, "%s%s%d", fname, SEP, i - 1) >= FILE_BUF_SIZE;
1451
1452 if (overflow)
1453 errno = ENAMETOOLONG;
1454 if (overflow || rename (from, to))
1455 {
1456 // The original file may not exist. In which case rename() will
1457 // return ENOENT. This is not a real error. We could make this better
1458 // by calling stat() first and making sure that the file exists.
1459 if (errno != ENOENT)
1460 logprintf (LOG_NOTQUIET, "Failed to rename %s to %s: (%d) %s\n",
1461 from, to, errno, strerror (errno));
1462 }
1463 }
1464
1465 overflow = (unsigned) snprintf (to, FILE_BUF_SIZE, "%s%s%d", fname, SEP, 1) >= FILE_BUF_SIZE;
1466 if (overflow)
1467 errno = ENAMETOOLONG;
1468 if (overflow || rename(fname, to))
1469 {
1470 if (errno != ENOENT)
1471 logprintf (LOG_NOTQUIET, "Failed to rename %s to %s: (%d) %s\n",
1472 from, to, errno, strerror (errno));
1473 }
1474
1475 #undef FILE_BUF_SIZE
1476 }
1477
1478 static bool no_proxy_match (const char *, const char **);
1479
1480 /* Return the URL of the proxy appropriate for url U. */
1481
1482 static char *
1483 getproxy (struct url *u)
1484 {
1485 char *proxy = NULL;
1486 char *rewritten_url;
1487
1488 if (!opt.use_proxy)
1489 return NULL;
1490 if (no_proxy_match (u->host, (const char **)opt.no_proxy))
1491 return NULL;
1492
1493 switch (u->scheme)
1494 {
1495 case SCHEME_HTTP:
1496 proxy = opt.http_proxy ? opt.http_proxy : getenv ("http_proxy");
1497 break;
1498 #ifdef HAVE_SSL
1499 case SCHEME_HTTPS:
1500 proxy = opt.https_proxy ? opt.https_proxy : getenv ("https_proxy");
1501 break;
1502 case SCHEME_FTPS:
1503 proxy = opt.ftp_proxy ? opt.ftp_proxy : getenv ("ftps_proxy");
1504 break;
1505 #endif
1506 case SCHEME_FTP:
1507 proxy = opt.ftp_proxy ? opt.ftp_proxy : getenv ("ftp_proxy");
1508 break;
1509 case SCHEME_INVALID:
1510 break;
1511 }
1512 if (!proxy || !*proxy)
1513 #ifdef HAVE_LIBPROXY
1514 {
1515 pxProxyFactory *pf = px_proxy_factory_new ();
1516 if (!pf)
1517 {
1518 DEBUGP (("Allocating memory for libproxy failed"));
1519 return NULL;
1520 }
1521
1522 DEBUGP (("asking libproxy about url '%s'\n", u->url));
1523 char **proxies = px_proxy_factory_get_proxies (pf, u->url);
1524 if (proxies)
1525 {
1526 if (proxies[0])
1527 {
1528 DEBUGP (("libproxy suggest to use '%s'\n", proxies[0]));
1529 if (strcmp (proxies[0], "direct://") != 0)
1530 {
1531 proxy = xstrdup (proxies[0]);
1532 DEBUGP (("libproxy setting to use '%s'\n", proxy));
1533 }
1534 }
1535
1536 px_proxy_factory_free_proxies (proxies);
1537 }
1538 px_proxy_factory_free (pf);
1539
1540 if (!proxy || !*proxy)
1541 return NULL;
1542 }
1543 #else
1544 return NULL;
1545 #endif
1546
1547 /* Handle shorthands. `rewritten_storage' is a kludge to allow
1548 getproxy() to return static storage. */
1549 rewritten_url = maybe_prepend_scheme (proxy);
1550 if (rewritten_url)
1551 return rewritten_url;
1552
1553 return strdup(proxy);
1554 }
1555
1556 /* Returns true if URL would be downloaded through a proxy. */
1557
1558 bool
1559 url_uses_proxy (struct url * u)
1560 {
1561 bool ret;
1562 char *proxy;
1563
1564 if (!u)
1565 return false;
1566 proxy = getproxy (u);
1567 ret = proxy != NULL;
1568 xfree (proxy);
1569 return ret;
1570 }
1571
1572 /* Should a host be accessed through proxy, concerning no_proxy? */
1573 static bool
1574 no_proxy_match (const char *host, const char **no_proxy)
1575 {
1576 if (!no_proxy)
1577 return false;
1578 else
1579 return sufmatch (no_proxy, host);
1580 }
1581
1582 /* Set the file parameter to point to the local file string. */
1583 void
1584 set_local_file (const char **file, const char *default_file)
1585 {
1586 if (opt.output_document)
1587 {
1588 if (output_stream_regular)
1589 *file = opt.output_document;
1590 }
1591 else
1592 *file = default_file;
1593 }
1594
1595 /* Return true for an input file's own URL, false otherwise. */
1596 bool
1597 input_file_url (const char *input_file)
1598 {
1599 static bool first = true;
1600
1601 if (input_file
1602 && url_has_scheme (input_file)
1603 && first)
1604 {
1605 first = false;
1606 return true;
1607 }
1608 else
1609 return false;
1610 }
1611
1612 #ifdef TESTING
1613
1614 #include <stdint.h>
1615 #include "../tests/unit-tests.h"
1616
1617 const char *
1618 test_retr_rate(void)
1619 {
1620 static const struct test {
1621 wgint bytes;
1622 double secs;
1623 const char *expected;
1624 } tests[] = {
1625 { 0, 1, "0.00 B/s" },
1626 { INT64_MAX, 1, "100 TB/s" },
1627 };
1628
1629 for (struct test *t = tests; t < tests+countof(tests); t++)
1630 {
1631 const char *result = retr_rate (t->bytes, t->secs);
1632
1633 if (strcmp(result,t->expected))
1634 return aprintf("%s: Expected '%s', got '%s'", __func__, t->expected, result);
1635 }
1636
1637 return NULL;
1638 }
1639
1640 #endif /* TESTING */