Debian Codegraph @

src/http.c

raw
1 /* HTTP support.
2 Copyright (C) 1996-2012, 2014-2015, 2018-2024 Free Software
3 Foundation, Inc.
4
5 This file is part of GNU Wget.
6
7 GNU Wget is free software; you can redistribute it and/or modify
8 it under the terms of the GNU General Public License as published by
9 the Free Software Foundation; either version 3 of the License, or
10 (at your option) any later version.
11
12 GNU Wget is distributed in the hope that it will be useful,
13 but WITHOUT ANY WARRANTY; without even the implied warranty of
14 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 GNU General Public License for more details.
16
17 You should have received a copy of the GNU General Public License
18 along with Wget. If not, see <http://www.gnu.org/licenses/>.
19
20 Additional permission under GNU GPL version 3 section 7
21
22 If you modify this program, or any covered work, by linking or
23 combining it with the OpenSSL project's OpenSSL library (or a
24 modified version of that library), containing parts covered by the
25 terms of the OpenSSL or SSLeay licenses, the Free Software Foundation
26 grants you additional permission to convey the resulting work.
27 Corresponding Source for a non-source form of such a combination
28 shall include the source code for the parts of OpenSSL used as well
29 as that of the covered work. */
30
31 #include "wget.h"
32
33 #include <stdio.h>
34 #include <stdlib.h>
35 #include <string.h>
36 #include <unistd.h>
37 #include <assert.h>
38 #include <errno.h>
39 #include <time.h>
40 #include <locale.h>
41 #include <fcntl.h>
42
43 #include "hash.h"
44 #include "http.h"
45 #include "hsts.h"
46 #include "utils.h"
47 #include "url.h"
48 #include "host.h"
49 #include "retr.h"
50 #include "connect.h"
51 #include "netrc.h"
52 #ifdef HAVE_SSL
53 # include "ssl.h"
54 #endif
55 #ifdef ENABLE_NTLM
56 # include "http-ntlm.h"
57 #endif
58 #include "cookies.h"
59 #include "md5.h"
60 #include "convert.h"
61 #include "spider.h"
62 #include "warc.h"
63 #include "c-strcase.h"
64 #include "version.h"
65 #include "xstrndup.h"
66 #ifdef HAVE_METALINK
67 # include "metalink.h"
68 #endif
69 #ifdef ENABLE_XATTR
70 #include "xattr.h"
71 #endif
72
73 #ifdef TESTING
74 #include "../tests/unit-tests.h"
75 #endif
76
77 #ifdef __VMS
78 # include "vms.h"
79 #endif /* def __VMS */
80
81
82 /* Forward decls. */
83 struct http_stat;
84 static char *create_authorization_line (const char *, const char *,
85 const char *, const char *,
86 const char *, bool *, uerr_t *);
87 static char *basic_authentication_encode (const char *, const char *);
88 static bool known_authentication_scheme_p (const char *, const char *);
89 static void ensure_extension (struct http_stat *, const char *, int *);
90 static void load_cookies (void);
91
92 static bool cookies_loaded_p;
93 static struct cookie_jar *wget_cookie_jar;
94
95 #define TEXTHTML_S "text/html"
96 #define TEXTXHTML_S "application/xhtml+xml"
97 #define TEXTCSS_S "text/css"
98
99 /* Some status code validation macros: */
100 #define H_10X(x) (((x) >= 100) && ((x) < 200))
101 #define H_20X(x) (((x) >= 200) && ((x) < 300))
102 #define H_PARTIAL(x) ((x) == HTTP_STATUS_PARTIAL_CONTENTS)
103 #define H_REDIRECTED(x) ((x) == HTTP_STATUS_MOVED_PERMANENTLY \
104 || (x) == HTTP_STATUS_MOVED_TEMPORARILY \
105 || (x) == HTTP_STATUS_SEE_OTHER \
106 || (x) == HTTP_STATUS_TEMPORARY_REDIRECT \
107 || (x) == HTTP_STATUS_PERMANENT_REDIRECT)
108
109 /* HTTP/1.0 status codes from RFC1945, provided for reference. */
110 /* Successful 2xx. */
111 #define HTTP_STATUS_OK 200
112 #define HTTP_STATUS_CREATED 201
113 #define HTTP_STATUS_ACCEPTED 202
114 #define HTTP_STATUS_NO_CONTENT 204
115 #define HTTP_STATUS_PARTIAL_CONTENTS 206
116
117 /* Redirection 3xx. */
118 #define HTTP_STATUS_MULTIPLE_CHOICES 300
119 #define HTTP_STATUS_MOVED_PERMANENTLY 301
120 #define HTTP_STATUS_MOVED_TEMPORARILY 302
121 #define HTTP_STATUS_SEE_OTHER 303 /* from HTTP/1.1 */
122 #define HTTP_STATUS_NOT_MODIFIED 304
123 #define HTTP_STATUS_TEMPORARY_REDIRECT 307 /* from HTTP/1.1 */
124 #define HTTP_STATUS_PERMANENT_REDIRECT 308 /* from HTTP/1.1 */
125
126 /* Client error 4xx. */
127 #define HTTP_STATUS_BAD_REQUEST 400
128 #define HTTP_STATUS_UNAUTHORIZED 401
129 #define HTTP_STATUS_FORBIDDEN 403
130 #define HTTP_STATUS_NOT_FOUND 404
131 #define HTTP_STATUS_RANGE_NOT_SATISFIABLE 416
132
133 /* Server errors 5xx. */
134 #define HTTP_STATUS_INTERNAL 500
135 #define HTTP_STATUS_NOT_IMPLEMENTED 501
136 #define HTTP_STATUS_BAD_GATEWAY 502
137 #define HTTP_STATUS_UNAVAILABLE 503
138 #define HTTP_STATUS_GATEWAY_TIMEOUT 504
139
140 enum rp {
141 rel_none, rel_name, rel_value, rel_both
142 };
143
144 struct request {
145 const char *method;
146 char *arg;
147
148 struct request_header {
149 char *name, *value;
150 enum rp release_policy;
151 } *headers;
152 int hcount, hcapacity;
153 };
154
155
156 /* Create a new, empty request. Set the request's method and its
157 arguments. METHOD should be a literal string (or it should outlive
158 the request) because it will not be freed. ARG will be freed by
159 request_free. */
160
161 static struct request *
162 request_new (const char *method, char *arg)
163 {
164 struct request *req = xnew0 (struct request);
165 req->hcapacity = 8;
166 req->headers = xnew_array (struct request_header, req->hcapacity);
167 req->method = method;
168 req->arg = arg;
169 return req;
170 }
171
172 /* Return the method string passed with the last call to
173 request_set_method. */
174
175 static const char *
176 request_method (const struct request *req)
177 {
178 return req->method;
179 }
180
181 /* Free one header according to the release policy specified with
182 request_set_header. */
183
184 static void
185 release_header (struct request_header *hdr)
186 {
187 switch (hdr->release_policy)
188 {
189 case rel_none:
190 break;
191 case rel_name:
192 xfree (hdr->name);
193 break;
194 case rel_value:
195 xfree (hdr->value);
196 break;
197 case rel_both:
198 xfree (hdr->name);
199 xfree (hdr->value);
200 break;
201 }
202 }
203
204 /* Set the request named NAME to VALUE. Specifically, this means that
205 a "NAME: VALUE\r\n" header line will be used in the request. If a
206 header with the same name previously existed in the request, its
207 value will be replaced by this one. A NULL value means do nothing.
208
209 RELEASE_POLICY determines whether NAME and VALUE should be released
210 (freed) with request_free. Allowed values are:
211
212 - rel_none - don't free NAME or VALUE
213 - rel_name - free NAME when done
214 - rel_value - free VALUE when done
215 - rel_both - free both NAME and VALUE when done
216
217 Setting release policy is useful when arguments come from different
218 sources. For example:
219
220 // Don't free literal strings!
221 request_set_header (req, "Pragma", "no-cache", rel_none);
222
223 // Don't free a global variable, we'll need it later.
224 request_set_header (req, "Referer", opt.referer, rel_none);
225
226 // Value freshly allocated, free it when done.
227 request_set_header (req, "Range",
228 aprintf ("bytes=%s-", number_to_static_string (hs->restval)),
229 rel_value);
230 */
231
232 static void
233 request_set_header (struct request *req, const char *name, const char *value,
234 enum rp release_policy)
235 {
236 struct request_header *hdr;
237 int i;
238
239 if (!value)
240 {
241 /* A NULL value is a no-op; if freeing the name is requested,
242 free it now to avoid leaks. */
243 if (release_policy == rel_name || release_policy == rel_both)
244 xfree (name);
245 return;
246 }
247
248 for (i = 0; i < req->hcount; i++)
249 {
250 hdr = &req->headers[i];
251 if (0 == c_strcasecmp (name, hdr->name))
252 {
253 /* Replace existing header. */
254 release_header (hdr);
255 hdr->name = (void *)name;
256 hdr->value = (void *)value;
257 hdr->release_policy = release_policy;
258 return;
259 }
260 }
261
262 /* Install new header. */
263
264 if (req->hcount >= req->hcapacity)
265 {
266 req->hcapacity <<= 1;
267 req->headers = xrealloc (req->headers, req->hcapacity * sizeof (*hdr));
268 }
269 hdr = &req->headers[req->hcount++];
270 hdr->name = (void *)name;
271 hdr->value = (void *)value;
272 hdr->release_policy = release_policy;
273 }
274
275 /* Like request_set_header, but sets the whole header line, as
276 provided by the user using the `--header' option. For example,
277 request_set_user_header (req, "Foo: bar") works just like
278 request_set_header (req, "Foo", "bar"). */
279
280 static void
281 request_set_user_header (struct request *req, const char *header)
282 {
283 const char *name, *p;
284
285 if (!(p = strchr (header, ':')))
286 return;
287
288 name = xstrndup(header, p - header);
289
290 ++p;
291 while (c_isspace (*p))
292 ++p;
293
294 request_set_header (req, name, p, rel_name);
295 }
296
297 /* Remove the header with specified name from REQ. Returns true if
298 the header was actually removed, false otherwise. */
299
300 static bool
301 request_remove_header (struct request *req, const char *name)
302 {
303 int i;
304 for (i = 0; i < req->hcount; i++)
305 {
306 struct request_header *hdr = &req->headers[i];
307 if (0 == c_strcasecmp (name, hdr->name))
308 {
309 release_header (hdr);
310 /* Move the remaining headers by one. */
311 if (i < req->hcount - 1)
312 memmove (hdr, hdr + 1, (req->hcount - i - 1) * sizeof (*hdr));
313 --req->hcount;
314 return true;
315 }
316 }
317 return false;
318 }
319
320 #define APPEND(p, str) do { \
321 int A_len = strlen (str); \
322 memcpy (p, str, A_len); \
323 p += A_len; \
324 } while (0)
325
326 /* Construct the request and write it to FD using fd_write.
327 If warc_tmp is set to a file pointer, the request string will
328 also be written to that file. */
329
330 static int
331 request_send (const struct request *req, int fd, FILE *warc_tmp)
332 {
333 char *request_string, *p;
334 int i, size, write_error;
335
336 /* Count the request size. */
337 size = 0;
338
339 /* METHOD " " ARG " " "HTTP/1.0" "\r\n" */
340 size += strlen (req->method) + 1 + strlen (req->arg) + 1 + 8 + 2;
341
342 for (i = 0; i < req->hcount; i++)
343 {
344 struct request_header *hdr = &req->headers[i];
345 /* NAME ": " VALUE "\r\n" */
346 size += strlen (hdr->name) + 2 + strlen (hdr->value) + 2;
347 }
348
349 /* "\r\n\0" */
350 size += 3;
351
352 p = request_string = xmalloc (size);
353
354 /* Generate the request. */
355
356 APPEND (p, req->method); *p++ = ' ';
357 APPEND (p, req->arg); *p++ = ' ';
358 memcpy (p, "HTTP/1.1\r\n", 10); p += 10;
359
360 for (i = 0; i < req->hcount; i++)
361 {
362 struct request_header *hdr = &req->headers[i];
363 APPEND (p, hdr->name);
364 *p++ = ':', *p++ = ' ';
365 APPEND (p, hdr->value);
366 *p++ = '\r', *p++ = '\n';
367 }
368
369 *p++ = '\r', *p++ = '\n', *p++ = '\0';
370 assert (p - request_string == size);
371
372 #undef APPEND
373
374 DEBUGP (("\n---request begin---\n%s---request end---\n", request_string));
375
376 /* Send the request to the server. */
377
378 write_error = fd_write (fd, request_string, size - 1, -1);
379 if (write_error < 0)
380 logprintf (LOG_VERBOSE, _("Failed writing HTTP request: %s.\n"),
381 fd_errstr (fd));
382 else if (warc_tmp != NULL)
383 {
384 /* Write a copy of the data to the WARC record. */
385 int warc_tmp_written = fwrite (request_string, 1, size - 1, warc_tmp);
386 if (warc_tmp_written != size - 1)
387 write_error = -2;
388 }
389 xfree (request_string);
390 return write_error;
391 }
392
393 /* Release the resources used by REQ.
394 It is safe to call it with a valid pointer to a NULL pointer.
395 It is not safe to call it with an invalid or NULL pointer. */
396
397 static void
398 request_free (struct request **req_ref)
399 {
400 int i;
401 struct request *req = *req_ref;
402
403 if (!req)
404 return;
405
406 xfree (req->arg);
407 for (i = 0; i < req->hcount; i++)
408 release_header (&req->headers[i]);
409 xfree (req->headers);
410 xfree (req);
411 *req_ref = NULL;
412 }
413
414 static struct hash_table *basic_authed_hosts;
415
416 /* Find out if this host has issued a Basic challenge yet; if so, give
417 * it the username, password. A temporary measure until we can get
418 * proper authentication in place. */
419
420 static bool
421 maybe_send_basic_creds (const char *hostname, const char *user,
422 const char *passwd, struct request *req)
423 {
424 bool do_challenge = false;
425
426 if (opt.auth_without_challenge)
427 {
428 DEBUGP (("Auth-without-challenge set, sending Basic credentials.\n"));
429 do_challenge = true;
430 }
431 else if (basic_authed_hosts
432 && hash_table_contains (basic_authed_hosts, hostname))
433 {
434 DEBUGP (("Found %s in basic_authed_hosts.\n", quote (hostname)));
435 do_challenge = true;
436 }
437 else
438 {
439 DEBUGP (("Host %s has not issued a general basic challenge.\n",
440 quote (hostname)));
441 }
442 if (do_challenge)
443 {
444 request_set_header (req, "Authorization",
445 basic_authentication_encode (user, passwd),
446 rel_value);
447 }
448 return do_challenge;
449 }
450
451 static void
452 register_basic_auth_host (const char *hostname)
453 {
454 if (!basic_authed_hosts)
455 {
456 basic_authed_hosts = make_nocase_string_hash_table (1);
457 }
458 if (!hash_table_contains (basic_authed_hosts, hostname))
459 {
460 hash_table_put (basic_authed_hosts, xstrdup (hostname), NULL);
461 DEBUGP (("Inserted %s into basic_authed_hosts\n", quote (hostname)));
462 }
463 }
464
465 /* Send the contents of FILE_NAME to SOCK. Make sure that exactly
466 PROMISED_SIZE bytes are sent over the wire -- if the file is
467 longer, read only that much; if the file is shorter, report an error.
468 If warc_tmp is set to a file pointer, the post data will
469 also be written to that file. */
470
471 static int
472 body_file_send (int sock, const char *file_name, wgint promised_size, FILE *warc_tmp)
473 {
474 static char chunk[8192];
475 wgint written = 0;
476 int write_error;
477 FILE *fp;
478
479 DEBUGP (("[writing BODY file %s ... ", file_name));
480
481 fp = fopen (file_name, "rb");
482 if (!fp)
483 return -1;
484 while (!feof (fp) && written < promised_size)
485 {
486 int towrite;
487 int length = fread (chunk, 1, sizeof (chunk), fp);
488 if (length == 0)
489 break;
490 towrite = MIN (promised_size - written, length);
491 write_error = fd_write (sock, chunk, towrite, -1);
492 if (write_error < 0)
493 {
494 fclose (fp);
495 return -1;
496 }
497 if (warc_tmp != NULL)
498 {
499 /* Write a copy of the data to the WARC record. */
500 int warc_tmp_written = fwrite (chunk, 1, towrite, warc_tmp);
501 if (warc_tmp_written != towrite)
502 {
503 fclose (fp);
504 return -2;
505 }
506 }
507 written += towrite;
508 }
509 fclose (fp);
510
511 /* If we've written less than was promised, report a (probably
512 nonsensical) error rather than break the promise. */
513 if (written < promised_size)
514 {
515 errno = EINVAL;
516 return -1;
517 }
518
519 assert (written == promised_size);
520 DEBUGP (("done]\n"));
521 return 0;
522 }
523
524 /* Determine whether [START, PEEKED + PEEKLEN) contains an empty line.
525 If so, return the pointer to the position after the line, otherwise
526 return NULL. This is used as callback to fd_read_hunk. The data
527 between START and PEEKED has been read and cannot be "unread"; the
528 data after PEEKED has only been peeked. */
529
530 static const char *
531 response_head_terminator (const char *start, const char *peeked, int peeklen)
532 {
533 const char *p, *end;
534
535 /* If at first peek, verify whether HUNK starts with "HTTP". If
536 not, this is a HTTP/0.9 request and we must bail out without
537 reading anything. */
538 if (start == peeked && 0 != memcmp (start, "HTTP", MIN (peeklen, 4)))
539 return start;
540
541 /* Look for "\n[\r]\n", and return the following position if found.
542 Start two chars before the current to cover the possibility that
543 part of the terminator (e.g. "\n\r") arrived in the previous
544 batch. */
545 p = peeked - start < 2 ? start : peeked - 2;
546 end = peeked + peeklen;
547
548 /* Check for \n\r\n or \n\n anywhere in [p, end-2). */
549 for (; p < end - 2; p++)
550 if (*p == '\n')
551 {
552 if (p[1] == '\r' && p[2] == '\n')
553 return p + 3;
554 else if (p[1] == '\n')
555 return p + 2;
556 }
557 /* p==end-2: check for \n\n directly preceding END. */
558 if (peeklen >= 2 && p[0] == '\n' && p[1] == '\n')
559 return p + 2;
560
561 return NULL;
562 }
563
564 /* The maximum size of a single HTTP response we care to read. Rather
565 than being a limit of the reader implementation, this limit
566 prevents Wget from slurping all available memory upon encountering
567 malicious or buggy server output, thus protecting the user. Define
568 it to 0 to remove the limit. */
569
570 #define HTTP_RESPONSE_MAX_SIZE 65536
571
572 /* Read the HTTP request head from FD and return it. The error
573 conditions are the same as with fd_read_hunk.
574
575 To support HTTP/0.9 responses, this function tries to make sure
576 that the data begins with "HTTP". If this is not the case, no data
577 is read and an empty request is returned, so that the remaining
578 data can be treated as body. */
579
580 static char *
581 read_http_response_head (int fd)
582 {
583 return fd_read_hunk (fd, response_head_terminator, 512,
584 HTTP_RESPONSE_MAX_SIZE);
585 }
586
587 struct response {
588 /* The response data. */
589 const char *data;
590
591 /* The array of pointers that indicate where each header starts.
592 For example, given this HTTP response:
593
594 HTTP/1.0 200 Ok
595 Description: some
596 text
597 Etag: x
598
599 The headers are located like this:
600
601 "HTTP/1.0 200 Ok\r\nDescription: some\r\n text\r\nEtag: x\r\n\r\n"
602 ^ ^ ^ ^
603 headers[0] headers[1] headers[2] headers[3]
604
605 I.e. headers[0] points to the beginning of the request,
606 headers[1] points to the end of the first header and the
607 beginning of the second one, etc. */
608
609 const char **headers;
610 };
611
612 /* Create a new response object from the text of the HTTP response,
613 available in HEAD. That text is automatically split into
614 constituent header lines for fast retrieval using
615 resp_header_*. */
616
617 static struct response *
618 resp_new (char *head)
619 {
620 char *hdr;
621 int count, size;
622
623 struct response *resp = xnew0 (struct response);
624 resp->data = head;
625
626 if (*head == '\0')
627 {
628 /* Empty head means that we're dealing with a headerless
629 (HTTP/0.9) response. In that case, don't set HEADERS at
630 all. */
631 return resp;
632 }
633
634 /* Split HEAD into header lines, so that resp_header_* functions
635 don't need to do this over and over again. */
636
637 size = count = 0;
638 hdr = head;
639 while (1)
640 {
641 DO_REALLOC (resp->headers, size, count + 1, const char *);
642 resp->headers[count++] = hdr;
643
644 /* Break upon encountering an empty line. */
645 if (!hdr[0] || (hdr[0] == '\r' && hdr[1] == '\n') || hdr[0] == '\n')
646 break;
647
648 /* Find the end of HDR, including continuations. */
649 for (;;)
650 {
651 char *end = strchr (hdr, '\n');
652
653 if (!end)
654 {
655 hdr += strlen (hdr);
656 break;
657 }
658
659 hdr = end + 1;
660
661 if (*hdr != ' ' && *hdr != '\t')
662 break;
663
664 // continuation, transform \r and \n into spaces
665 *end = ' ';
666 if (end > head && end[-1] == '\r')
667 end[-1] = ' ';
668 }
669 }
670 DO_REALLOC (resp->headers, size, count + 1, const char *);
671 resp->headers[count] = NULL;
672
673 return resp;
674 }
675
676 /* Locate the header named NAME in the request data, starting with
677 position START. This allows the code to loop through the request
678 data, filtering for all requests of a given name. Returns the
679 found position, or -1 for failure. The code that uses this
680 function typically looks like this:
681
682 for (pos = 0; (pos = resp_header_locate (...)) != -1; pos++)
683 ... do something with header ...
684
685 If you only care about one header, use resp_header_get instead of
686 this function. */
687
688 static int
689 resp_header_locate (const struct response *resp, const char *name, int start,
690 const char **begptr, const char **endptr)
691 {
692 int i;
693 const char **headers = resp->headers;
694 int name_len;
695
696 if (!headers || !headers[1])
697 return -1;
698
699 name_len = strlen (name);
700 if (start > 0)
701 i = start;
702 else
703 i = 1;
704
705 for (; headers[i + 1]; i++)
706 {
707 const char *b = headers[i];
708 const char *e = headers[i + 1];
709 if (e - b > name_len
710 && b[name_len] == ':'
711 && 0 == c_strncasecmp (b, name, name_len))
712 {
713 b += name_len + 1;
714 while (b < e && c_isspace (*b))
715 ++b;
716 while (b < e && c_isspace (e[-1]))
717 --e;
718 *begptr = b;
719 *endptr = e;
720 return i;
721 }
722 }
723 return -1;
724 }
725
726 /* Find and retrieve the header named NAME in the request data. If
727 found, set *BEGPTR to its starting, and *ENDPTR to its ending
728 position, and return true. Otherwise return false.
729
730 This function is used as a building block for resp_header_copy
731 and resp_header_strdup. */
732
733 static bool
734 resp_header_get (const struct response *resp, const char *name,
735 const char **begptr, const char **endptr)
736 {
737 int pos = resp_header_locate (resp, name, 0, begptr, endptr);
738 return pos != -1;
739 }
740
741 /* Copy the response header named NAME to buffer BUF, no longer than
742 BUFSIZE (BUFSIZE includes the terminating 0). If the header
743 exists, true is returned, false otherwise. If there should be no
744 limit on the size of the header, use resp_header_strdup instead.
745
746 If BUFSIZE is 0, no data is copied, but the boolean indication of
747 whether the header is present is still returned. */
748
749 static bool
750 resp_header_copy (const struct response *resp, const char *name,
751 char *buf, int bufsize)
752 {
753 const char *b, *e;
754 if (!resp_header_get (resp, name, &b, &e))
755 return false;
756 if (bufsize)
757 {
758 int len = MIN (e - b, bufsize - 1);
759 memcpy (buf, b, len);
760 buf[len] = '\0';
761 }
762 return true;
763 }
764
765 /* Return the value of header named NAME in RESP, allocated with
766 malloc. If such a header does not exist in RESP, return NULL. */
767
768 static char *
769 resp_header_strdup (const struct response *resp, const char *name)
770 {
771 const char *b, *e;
772 if (!resp_header_get (resp, name, &b, &e))
773 return NULL;
774 return strdupdelim (b, e);
775 }
776
777 /* Parse the HTTP status line, which is of format:
778
779 HTTP-Version SP Status-Code SP Reason-Phrase
780
781 The function returns the status-code, or -1 if the status line
782 appears malformed. The pointer to "reason-phrase" message is
783 returned in *MESSAGE. */
784
785 static int
786 resp_status (const struct response *resp, char **message)
787 {
788 int status;
789 const char *p, *end;
790
791 if (!resp->headers)
792 {
793 /* For a HTTP/0.9 response, assume status 200. */
794 if (message)
795 *message = xstrdup (_("No headers, assuming HTTP/0.9"));
796 return 200;
797 }
798
799 p = resp->headers[0];
800 end = resp->headers[1];
801
802 if (!end)
803 return -1;
804
805 /* "HTTP" */
806 if (end - p < 4 || 0 != strncmp (p, "HTTP", 4))
807 return -1;
808 p += 4;
809
810 /* Match the HTTP version. This is optional because Gnutella
811 servers have been reported to not specify HTTP version. */
812 if (p < end && *p == '/')
813 {
814 ++p;
815 while (p < end && c_isdigit (*p))
816 ++p;
817 if (p < end && *p == '.')
818 ++p;
819 while (p < end && c_isdigit (*p))
820 ++p;
821 }
822
823 while (p < end && c_isspace (*p))
824 ++p;
825 if (end - p < 3 || !c_isdigit (p[0]) || !c_isdigit (p[1]) || !c_isdigit (p[2]))
826 return -1;
827
828 status = 100 * (p[0] - '0') + 10 * (p[1] - '0') + (p[2] - '0');
829 p += 3;
830
831 if (message)
832 {
833 while (p < end && c_isspace (*p))
834 ++p;
835 while (p < end && c_isspace (end[-1]))
836 --end;
837 *message = strdupdelim (p, end);
838 }
839
840 return status;
841 }
842
843 /* Release the resources used by RESP.
844 It is safe to call it with a valid pointer to a NULL pointer.
845 It is not safe to call it with a invalid or NULL pointer. */
846
847 static void
848 resp_free (struct response **resp_ref)
849 {
850 struct response *resp = *resp_ref;
851
852 if (!resp)
853 return;
854
855 xfree (resp->headers);
856 xfree (resp);
857
858 *resp_ref = NULL;
859 }
860
861 /* Print a single line of response, the characters [b, e). We tried
862 getting away with
863 logprintf (LOG_VERBOSE, "%s%.*s\n", prefix, (int) (e - b), b);
864 but that failed to escape the non-printable characters and, in fact,
865 caused crashes in UTF-8 locales. */
866
867 static void
868 print_response_line (const char *prefix, const char *b, const char *e)
869 {
870 char buf[1024], *copy;
871 size_t len = e - b;
872
873 if (len < sizeof (buf))
874 copy = buf;
875 else
876 copy = xmalloc(len + 1);
877
878 memcpy(copy, b, len);
879 copy[len] = 0;
880
881 logprintf (LOG_ALWAYS, "%s%s\n", prefix,
882 quotearg_style (escape_quoting_style, copy));
883
884 if (copy != buf)
885 xfree (copy);
886 }
887
888 /* Print the server response, line by line, omitting the trailing CRLF
889 from individual header lines, and prefixed with PREFIX. */
890
891 static void
892 print_server_response (const struct response *resp, const char *prefix)
893 {
894 int i;
895 if (!resp->headers)
896 return;
897 for (i = 0; resp->headers[i + 1]; i++)
898 {
899 const char *b = resp->headers[i];
900 const char *e = resp->headers[i + 1];
901 /* Skip CRLF */
902 if (b < e && e[-1] == '\n')
903 --e;
904 if (b < e && e[-1] == '\r')
905 --e;
906 print_response_line (prefix, b, e);
907 }
908 }
909
910 /* Parse the `Content-Range' header and extract the information it
911 contains. Returns true if successful, false otherwise. */
912 static bool
913 parse_content_range (const char *hdr, wgint *first_byte_ptr,
914 wgint *last_byte_ptr, wgint *entity_length_ptr)
915 {
916 wgint num;
917
918 /* Ancient versions of Netscape proxy server, presumably predating
919 rfc2068, sent out `Content-Range' without the "bytes"
920 specifier. */
921 if (0 == strncasecmp (hdr, "bytes", 5))
922 {
923 hdr += 5;
924 /* "JavaWebServer/1.1.1" sends "bytes: x-y/z", contrary to the
925 HTTP spec. */
926 if (*hdr == ':')
927 ++hdr;
928 while (c_isspace (*hdr))
929 ++hdr;
930 if (!*hdr)
931 return false;
932 }
933 if (!c_isdigit (*hdr))
934 return false;
935 for (num = 0; c_isdigit (*hdr); hdr++)
936 num = 10 * num + (*hdr - '0');
937 if (*hdr != '-' || !c_isdigit (*(hdr + 1)))
938 return false;
939 *first_byte_ptr = num;
940 ++hdr;
941 for (num = 0; c_isdigit (*hdr); hdr++)
942 num = 10 * num + (*hdr - '0');
943 if (*hdr != '/')
944 return false;
945 *last_byte_ptr = num;
946 if (!(c_isdigit (*(hdr + 1)) || *(hdr + 1) == '*'))
947 return false;
948 if (*last_byte_ptr < *first_byte_ptr)
949 return false;
950 ++hdr;
951 if (*hdr == '*')
952 num = -1;
953 else
954 for (num = 0; c_isdigit (*hdr); hdr++)
955 num = 10 * num + (*hdr - '0');
956 *entity_length_ptr = num;
957 if ((*entity_length_ptr <= *last_byte_ptr) && *entity_length_ptr != -1)
958 return false;
959 return true;
960 }
961
962 /* Read the body of the request, but don't store it anywhere and don't
963 display a progress gauge. This is useful for reading the bodies of
964 administrative responses to which we will soon issue another
965 request. The response is not useful to the user, but reading it
966 allows us to continue using the same connection to the server.
967
968 If reading fails, false is returned, true otherwise. In debug
969 mode, the body is displayed for debugging purposes. */
970
971 static bool
972 skip_short_body (int fd, wgint contlen, bool chunked)
973 {
974 enum {
975 SKIP_SIZE = 512, /* size of the download buffer */
976 SKIP_THRESHOLD = 4096 /* the largest size we read */
977 };
978 wgint remaining_chunk_size = 0;
979 char dlbuf[SKIP_SIZE + 1];
980 dlbuf[SKIP_SIZE] = '\0'; /* so DEBUGP can safely print it */
981
982 /* If the body is too large, it makes more sense to simply close the
983 connection than to try to read the body. */
984 if (contlen > SKIP_THRESHOLD)
985 return false;
986
987 while (contlen > 0 || chunked)
988 {
989 int ret;
990 if (chunked)
991 {
992 if (remaining_chunk_size == 0)
993 {
994 char *line = fd_read_line (fd);
995 char *endl;
996 if (line == NULL)
997 break;
998
999 remaining_chunk_size = strtol (line, &endl, 16);
1000 xfree (line);
1001
1002 if (remaining_chunk_size < 0)
1003 return false;
1004
1005 if (remaining_chunk_size == 0)
1006 {
1007 line = fd_read_line (fd);
1008 xfree (line);
1009 break;
1010 }
1011 }
1012
1013 contlen = MIN (remaining_chunk_size, SKIP_SIZE);
1014 }
1015
1016 DEBUGP (("Skipping %s bytes of body: [", number_to_static_string (contlen)));
1017
1018 ret = fd_read (fd, dlbuf, MIN (contlen, SKIP_SIZE), -1);
1019 if (ret <= 0)
1020 {
1021 /* Don't normally report the error since this is an
1022 optimization that should be invisible to the user. */
1023 DEBUGP (("] aborting (%s).\n",
1024 ret < 0 ? fd_errstr (fd) : "EOF received"));
1025 return false;
1026 }
1027 contlen -= ret;
1028
1029 if (chunked)
1030 {
1031 remaining_chunk_size -= ret;
1032 if (remaining_chunk_size == 0)
1033 {
1034 char *line = fd_read_line (fd);
1035 if (line == NULL)
1036 return false;
1037 else
1038 xfree (line);
1039 }
1040 }
1041
1042 /* Safe even if %.*s bogusly expects terminating \0 because
1043 we've zero-terminated dlbuf above. */
1044 DEBUGP (("%.*s", ret, dlbuf));
1045 }
1046
1047 DEBUGP (("] done.\n"));
1048 return true;
1049 }
1050
1051 #define NOT_RFC2231 0
1052 #define RFC2231_NOENCODING 1
1053 #define RFC2231_ENCODING 2
1054
1055 /* extract_param extracts the parameter name into NAME.
1056 However, if the parameter name is in RFC2231 format then
1057 this function adjusts NAME by stripping of the trailing
1058 characters that are not part of the name but are present to
1059 indicate the presence of encoding information in the value
1060 or a fragment of a long parameter value
1061 */
1062 static int
1063 modify_param_name (param_token *name)
1064 {
1065 const char *delim1 = memchr (name->b, '*', name->e - name->b);
1066 const char *delim2 = memrchr (name->b, '*', name->e - name->b);
1067
1068 int result;
1069
1070 if (delim1 == NULL)
1071 {
1072 result = NOT_RFC2231;
1073 }
1074 else if (delim1 == delim2)
1075 {
1076 if ((name->e - 1) == delim1)
1077 {
1078 result = RFC2231_ENCODING;
1079 }
1080 else
1081 {
1082 result = RFC2231_NOENCODING;
1083 }
1084 name->e = delim1;
1085 }
1086 else
1087 {
1088 name->e = delim1;
1089 result = RFC2231_ENCODING;
1090 }
1091 return result;
1092 }
1093
1094 /* extract_param extract the parameter value into VALUE.
1095 Like modify_param_name this function modifies VALUE by
1096 stripping off the encoding information from the actual value
1097 */
1098 static void
1099 modify_param_value (param_token *value, int encoding_type )
1100 {
1101 if (encoding_type == RFC2231_ENCODING)
1102 {
1103 const char *delim = memrchr (value->b, '\'', value->e - value->b);
1104 if (delim != NULL)
1105 {
1106 value->b = (delim+1);
1107 }
1108 }
1109 }
1110
1111 /* Extract a parameter from the string (typically an HTTP header) at
1112 **SOURCE and advance SOURCE to the next parameter. Return false
1113 when there are no more parameters to extract. The name of the
1114 parameter is returned in NAME, and the value in VALUE. If the
1115 parameter has no value, the token's value is zeroed out.
1116
1117 For example, if *SOURCE points to the string "attachment;
1118 filename=\"foo bar\"", the first call to this function will return
1119 the token named "attachment" and no value, and the second call will
1120 return the token named "filename" and value "foo bar". The third
1121 call will return false, indicating no more valid tokens.
1122
1123 is_url_encoded is an out parameter. If not NULL, a boolean value will be
1124 stored into it, letting the caller know whether or not the extracted value is
1125 URL-encoded. The caller can then decode it with url_unescape(), which however
1126 performs decoding in-place. URL-encoding is used by RFC 2231 to support
1127 non-US-ASCII characters in HTTP header values. */
1128
1129 bool
1130 extract_param (const char **source, param_token *name, param_token *value,
1131 char separator, bool *is_url_encoded)
1132 {
1133 const char *p = *source;
1134 int param_type;
1135 if (is_url_encoded)
1136 *is_url_encoded = false; /* initializing the out parameter */
1137
1138 while (c_isspace (*p)) ++p;
1139 if (!*p)
1140 {
1141 *source = p;
1142 return false; /* no error; nothing more to extract */
1143 }
1144
1145 /* Extract name. */
1146 name->b = p;
1147 while (*p && !c_isspace (*p) && *p != '=' && *p != separator) ++p;
1148 name->e = p;
1149 if (name->b == name->e)
1150 return false; /* empty name: error */
1151 while (c_isspace (*p)) ++p;
1152 if (*p == separator || !*p) /* no value */
1153 {
1154 xzero (*value);
1155 if (*p == separator) ++p;
1156 *source = p;
1157 return true;
1158 }
1159 if (*p != '=')
1160 return false; /* error */
1161
1162 /* *p is '=', extract value */
1163 ++p;
1164 while (c_isspace (*p)) ++p;
1165 if (*p == '"') /* quoted */
1166 {
1167 value->b = ++p;
1168 while (*p && *p != '"') ++p;
1169 if (!*p)
1170 return false;
1171 value->e = p++;
1172 /* Currently at closing quote; find the end of param. */
1173 while (c_isspace (*p)) ++p;
1174 while (*p && *p != separator) ++p;
1175 if (*p == separator)
1176 ++p;
1177 else if (*p)
1178 /* garbage after closed quote, e.g. foo="bar"baz */
1179 return false;
1180 }
1181 else /* unquoted */
1182 {
1183 value->b = p;
1184 while (*p && *p != separator) ++p;
1185 value->e = p;
1186 while (value->e != value->b && c_isspace (value->e[-1]))
1187 --value->e;
1188 if (*p == separator) ++p;
1189 }
1190 *source = p;
1191
1192 param_type = modify_param_name (name);
1193 if (param_type != NOT_RFC2231)
1194 {
1195 if (param_type == RFC2231_ENCODING && is_url_encoded)
1196 *is_url_encoded = true;
1197 modify_param_value (value, param_type);
1198 }
1199 return true;
1200 }
1201
1202 #undef NOT_RFC2231
1203 #undef RFC2231_NOENCODING
1204 #undef RFC2231_ENCODING
1205
1206 /* Appends the string represented by VALUE to FILENAME */
1207
1208 static void
1209 append_value_to_filename (char **filename, param_token const * const value,
1210 bool is_url_encoded)
1211 {
1212 int original_length = strlen (*filename);
1213 int new_length = strlen (*filename) + (value->e - value->b);
1214 *filename = xrealloc (*filename, new_length+1);
1215 memcpy (*filename + original_length, value->b, (value->e - value->b));
1216 (*filename)[new_length] = '\0';
1217 if (is_url_encoded)
1218 url_unescape (*filename + original_length);
1219 }
1220
1221 /* Parse the contents of the `Content-Disposition' header, extracting
1222 the information useful to Wget. Content-Disposition is a header
1223 borrowed from MIME; when used in HTTP, it typically serves for
1224 specifying the desired file name of the resource. For example:
1225
1226 Content-Disposition: attachment; filename="flora.jpg"
1227
1228 Wget will skip the tokens it doesn't care about, such as
1229 "attachment" in the previous example; it will also skip other
1230 unrecognized params. If the header is syntactically correct and
1231 contains a file name, a copy of the file name is stored in
1232 *filename and true is returned. Otherwise, the function returns
1233 false.
1234
1235 The file name is stripped of directory components and must not be
1236 empty.
1237
1238 Historically, this function returned filename prefixed with opt.dir_prefix,
1239 now that logic is handled by the caller, new code should pay attention,
1240 changed by crq, Sep 2010.
1241
1242 */
1243 static bool
1244 parse_content_disposition (const char *hdr, char **filename)
1245 {
1246 param_token name, value;
1247 bool is_url_encoded = false;
1248
1249 char *encodedFilename = NULL;
1250 char *unencodedFilename = NULL;
1251 for ( ; extract_param (&hdr, &name, &value, ';', &is_url_encoded);
1252 is_url_encoded = false)
1253 {
1254 int isFilename = BOUNDED_EQUAL_NO_CASE (name.b, name.e, "filename");
1255 if ( isFilename && value.b != NULL)
1256 {
1257 /* Make the file name begin at the last slash or backslash. */
1258 bool isEncodedFilename;
1259 char **outFilename;
1260 const char *last_slash = memrchr (value.b, '/', value.e - value.b);
1261 const char *last_bs = memrchr (value.b, '\\', value.e - value.b);
1262 if (last_slash && last_bs)
1263 value.b = 1 + MAX (last_slash, last_bs);
1264 else if (last_slash || last_bs)
1265 value.b = 1 + (last_slash ? last_slash : last_bs);
1266 if (value.b == value.e)
1267 continue;
1268
1269 /* Check if the name is "filename*" as specified in RFC 6266.
1270 * Since "filename" could be broken up as "filename*N" (RFC 2231),
1271 * a check is needed to make sure this is not the case */
1272 isEncodedFilename = *name.e == '*' && !c_isdigit (*(name.e + 1));
1273 outFilename = isEncodedFilename ? &encodedFilename
1274 : &unencodedFilename;
1275 if (*outFilename)
1276 append_value_to_filename (outFilename, &value, is_url_encoded);
1277 else
1278 {
1279 *outFilename = strdupdelim (value.b, value.e);
1280 if (is_url_encoded)
1281 url_unescape (*outFilename);
1282 }
1283 }
1284 }
1285 if (encodedFilename)
1286 {
1287 xfree (unencodedFilename);
1288 *filename = encodedFilename;
1289 }
1290 else
1291 {
1292 xfree (encodedFilename);
1293 *filename = unencodedFilename;
1294 }
1295 if (*filename)
1296 return true;
1297 else
1298 return false;
1299 }
1300
1301 #ifdef HAVE_HSTS
1302 static bool
1303 parse_strict_transport_security (const char *header, int64_t *max_age, bool *include_subdomains)
1304 {
1305 param_token name, value;
1306 const char *c_max_age = NULL;
1307 bool is = false; /* includeSubDomains */
1308 bool is_url_encoded = false;
1309 bool success = false;
1310
1311 if (header)
1312 {
1313 /* Process the STS header. Keys should be matched case-insensitively. */
1314 for (; extract_param (&header, &name, &value, ';', &is_url_encoded); is_url_encoded = false)
1315 {
1316 if (BOUNDED_EQUAL_NO_CASE (name.b, name.e, "max-age"))
1317 {
1318 xfree (c_max_age);
1319 c_max_age = strdupdelim (value.b, value.e);
1320 }
1321 else if (BOUNDED_EQUAL_NO_CASE (name.b, name.e, "includeSubDomains"))
1322 is = true;
1323 }
1324
1325 /* pass the parsed values over */
1326 if (c_max_age)
1327 {
1328 /* If the string value goes out of a long's bounds, strtol() will return LONG_MIN or LONG_MAX.
1329 * In theory, the HSTS engine should be able to handle it.
1330 * Also, time_t is normally defined as a long, so this should not break.
1331 */
1332 if (max_age)
1333 *max_age = (int64_t) strtoll (c_max_age, NULL, 10);
1334 if (include_subdomains)
1335 *include_subdomains = is;
1336
1337 DEBUGP (("Parsed Strict-Transport-Security max-age = %s, includeSubDomains = %s\n",
1338 c_max_age, (is ? "true" : "false")));
1339
1340 xfree (c_max_age);
1341 success = true;
1342 }
1343 else
1344 {
1345 /* something weird happened */
1346 logprintf (LOG_VERBOSE, "Could not parse Strict-Transport-Security header\n");
1347 success = false;
1348 }
1349 }
1350
1351 return success;
1352 }
1353 #endif
1354
1355 /* Persistent connections. Currently, we cache the most recently used
1356 connection as persistent, provided that the HTTP server agrees to
1357 make it such. The persistence data is stored in the variables
1358 below. Ideally, it should be possible to cache an arbitrary fixed
1359 number of these connections. */
1360
1361 /* Whether a persistent connection is active. */
1362 static bool pconn_active;
1363
1364 static struct {
1365 /* The socket of the connection. */
1366 int socket;
1367
1368 /* Host and port of the currently active persistent connection. */
1369 char *host;
1370 int port;
1371
1372 /* Whether a ssl handshake has occurred on this connection. */
1373 bool ssl;
1374
1375 /* Whether the connection was authorized. This is only done by
1376 NTLM, which authorizes *connections* rather than individual
1377 requests. (That practice is peculiar for HTTP, but it is a
1378 useful optimization.) */
1379 bool authorized;
1380
1381 #ifdef ENABLE_NTLM
1382 /* NTLM data of the current connection. */
1383 struct ntlmdata ntlm;
1384 #endif
1385 } pconn;
1386
1387 /* Mark the persistent connection as invalid and free the resources it
1388 uses. This is used by the CLOSE_* macros after they forcefully
1389 close a registered persistent connection. */
1390
1391 static void
1392 invalidate_persistent (void)
1393 {
1394 DEBUGP (("Disabling further reuse of socket %d.\n", pconn.socket));
1395 pconn_active = false;
1396 fd_close (pconn.socket);
1397 xfree (pconn.host);
1398 xzero (pconn);
1399 }
1400
1401 /* Register FD, which should be a TCP/IP connection to HOST:PORT, as
1402 persistent. This will enable someone to use the same connection
1403 later. In the context of HTTP, this must be called only AFTER the
1404 response has been received and the server has promised that the
1405 connection will remain alive.
1406
1407 If a previous connection was persistent, it is closed. */
1408
1409 static void
1410 register_persistent (const char *host, int port, int fd, bool ssl)
1411 {
1412 if (pconn_active)
1413 {
1414 if (pconn.socket == fd)
1415 {
1416 /* The connection FD is already registered. */
1417 return;
1418 }
1419 else
1420 {
1421 /* The old persistent connection is still active; close it
1422 first. This situation arises whenever a persistent
1423 connection exists, but we then connect to a different
1424 host, and try to register a persistent connection to that
1425 one. */
1426 invalidate_persistent ();
1427 }
1428 }
1429
1430 pconn_active = true;
1431 pconn.socket = fd;
1432 pconn.host = xstrdup (host);
1433 pconn.port = port;
1434 pconn.ssl = ssl;
1435 pconn.authorized = false;
1436
1437 DEBUGP (("Registered socket %d for persistent reuse.\n", fd));
1438 }
1439
1440 /* Return true if a persistent connection is available for connecting
1441 to HOST:PORT. */
1442
1443 static bool
1444 persistent_available_p (const char *host, int port, bool ssl,
1445 bool *host_lookup_failed)
1446 {
1447 /* First, check whether a persistent connection is active at all. */
1448 if (!pconn_active)
1449 return false;
1450
1451 /* If we want SSL and the last connection wasn't or vice versa,
1452 don't use it. Checking for host and port is not enough because
1453 HTTP and HTTPS can apparently coexist on the same port. */
1454 if (ssl != pconn.ssl)
1455 return false;
1456
1457 /* If we're not connecting to the same port, we're not interested. */
1458 if (port != pconn.port)
1459 return false;
1460
1461 /* If the host is the same, we're in business. If not, there is
1462 still hope -- read below. */
1463 if (0 != strcasecmp (host, pconn.host))
1464 {
1465 /* Check if pconn.socket is talking to HOST under another name.
1466 This happens often when both sites are virtual hosts
1467 distinguished only by name and served by the same network
1468 interface, and hence the same web server (possibly set up by
1469 the ISP and serving many different web sites). This
1470 admittedly unconventional optimization does not contradict
1471 HTTP and works well with popular server software. */
1472
1473 bool found;
1474 ip_address ip;
1475 struct address_list *al;
1476
1477 if (ssl)
1478 /* Don't try to talk to two different SSL sites over the same
1479 secure connection! (Besides, it's not clear that
1480 name-based virtual hosting is even possible with SSL.) */
1481 return false;
1482
1483 /* If pconn.socket's peer is one of the IP addresses HOST
1484 resolves to, pconn.socket is for all intents and purposes
1485 already talking to HOST. */
1486
1487 if (!socket_ip_address (pconn.socket, &ip, ENDPOINT_PEER))
1488 {
1489 /* Can't get the peer's address -- something must be very
1490 wrong with the connection. */
1491 invalidate_persistent ();
1492 return false;
1493 }
1494 al = lookup_host (host, 0);
1495 if (!al)
1496 {
1497 *host_lookup_failed = true;
1498 return false;
1499 }
1500
1501 found = address_list_contains (al, &ip);
1502 address_list_release (al);
1503
1504 if (!found)
1505 return false;
1506
1507 /* The persistent connection's peer address was found among the
1508 addresses HOST resolved to; therefore, pconn.sock is in fact
1509 already talking to HOST -- no need to reconnect. */
1510 }
1511
1512 /* Finally, check whether the connection is still open. This is
1513 important because most servers implement liberal (short) timeout
1514 on persistent connections. Wget can of course always reconnect
1515 if the connection doesn't work out, but it's nicer to know in
1516 advance. This test is a logical followup of the first test, but
1517 is "expensive" and therefore placed at the end of the list.
1518
1519 (Current implementation of test_socket_open has a nice side
1520 effect that it treats sockets with pending data as "closed".
1521 This is exactly what we want: if a broken server sends message
1522 body in response to HEAD, or if it sends more than conent-length
1523 data, we won't reuse the corrupted connection.) */
1524
1525 if (!test_socket_open (pconn.socket))
1526 {
1527 /* Oops, the socket is no longer open. Now that we know that,
1528 let's invalidate the persistent connection before returning
1529 0. */
1530 invalidate_persistent ();
1531 return false;
1532 }
1533
1534 return true;
1535 }
1536
1537 /* The idea behind these two CLOSE macros is to distinguish between
1538 two cases: one when the job we've been doing is finished, and we
1539 want to close the connection and leave, and two when something is
1540 seriously wrong and we're closing the connection as part of
1541 cleanup.
1542
1543 In case of keep_alive, CLOSE_FINISH should leave the connection
1544 open, while CLOSE_INVALIDATE should still close it.
1545
1546 Note that the semantics of the flag `keep_alive' is "this
1547 connection *will* be reused (the server has promised not to close
1548 the connection once we're done)", while the semantics of
1549 `pc_active_p && (fd) == pc_last_fd' is "we're *now* using an
1550 active, registered connection". */
1551
1552 #define CLOSE_FINISH(fd) do { \
1553 if (!keep_alive) \
1554 { \
1555 if (pconn_active && (fd) == pconn.socket) \
1556 invalidate_persistent (); \
1557 else \
1558 fd_close (fd); \
1559 fd = -1; \
1560 } \
1561 } while (0)
1562
1563 #define CLOSE_INVALIDATE(fd) do { \
1564 if (pconn_active && (fd) == pconn.socket) \
1565 invalidate_persistent (); \
1566 else \
1567 fd_close (fd); \
1568 fd = -1; \
1569 } while (0)
1570
1571 typedef enum
1572 {
1573 ENC_INVALID = -1, /* invalid encoding */
1574 ENC_NONE = 0, /* no special encoding */
1575 ENC_GZIP, /* gzip compression */
1576 ENC_DEFLATE, /* deflate compression */
1577 ENC_COMPRESS, /* compress compression */
1578 ENC_BROTLI /* brotli compression */
1579 } encoding_t;
1580
1581 struct http_stat
1582 {
1583 wgint len; /* received length */
1584 wgint contlen; /* expected length */
1585 wgint restval; /* the restart value */
1586 int res; /* the result of last read */
1587 char *rderrmsg; /* error message from read error */
1588 char *newloc; /* new location (redirection) */
1589 char *remote_time; /* remote time-stamp string */
1590 char *error; /* textual HTTP error */
1591 int statcode; /* status code */
1592 char *message; /* status message */
1593 wgint rd_size; /* amount of data read from socket */
1594 double dltime; /* time it took to download the data */
1595 const char *referer; /* value of the referer header. */
1596 char *local_file; /* local file name. */
1597 bool existence_checked; /* true if we already checked for a file's
1598 existence after having begun to download
1599 (needed in gethttp for when connection is
1600 interrupted/restarted. */
1601 bool timestamp_checked; /* true if pre-download time-stamping checks
1602 * have already been performed */
1603 char *orig_file_name; /* name of file to compare for time-stamping
1604 * (might be != local_file if -K is set) */
1605 wgint orig_file_size; /* size of file to compare for time-stamping */
1606 time_t orig_file_tstamp; /* time-stamp of file to compare for
1607 * time-stamping */
1608 #ifdef HAVE_METALINK
1609 metalink_t *metalink;
1610 #endif
1611
1612 encoding_t local_encoding; /* the encoding of the local file */
1613 encoding_t remote_encoding; /* the encoding of the remote file */
1614
1615 bool temporary; /* downloading a temporary file */
1616 };
1617
1618 static void
1619 free_hstat (struct http_stat *hs)
1620 {
1621 xfree (hs->newloc);
1622 xfree (hs->remote_time);
1623 xfree (hs->error);
1624 xfree (hs->rderrmsg);
1625 xfree (hs->local_file);
1626 xfree (hs->orig_file_name);
1627 xfree (hs->message);
1628 #ifdef HAVE_METALINK
1629 metalink_delete (hs->metalink);
1630 hs->metalink = NULL;
1631 #endif
1632 }
1633
1634 static void
1635 get_file_flags (const char *filename, int *dt)
1636 {
1637 logprintf (LOG_VERBOSE, _("\
1638 File %s already there; not retrieving.\n\n"), quote (filename));
1639 /* If the file is there, we suppose it's retrieved OK. */
1640 *dt |= RETROKF;
1641
1642 /* #### Bogusness alert. */
1643 /* If its suffix is "html" or "htm" or similar, assume text/html. */
1644 if (has_html_suffix_p (filename))
1645 *dt |= TEXTHTML;
1646 }
1647
1648 /* Download the response body from the socket and writes it to
1649 an output file. The headers have already been read from the
1650 socket. If WARC is enabled, the response body will also be
1651 written to a WARC response record.
1652
1653 hs, contlen, contrange, chunked_transfer_encoding and url are
1654 parameters from the gethttp method. fp is a pointer to the
1655 output file.
1656
1657 url, warc_timestamp_str, warc_request_uuid, warc_ip, type
1658 and statcode will be saved in the headers of the WARC record.
1659 The head parameter contains the HTTP headers of the response.
1660
1661 If fp is NULL and WARC is enabled, the response body will be
1662 written only to the WARC file. If WARC is disabled and fp
1663 is a file pointer, the data will be written to the file.
1664 If fp is a file pointer and WARC is enabled, the body will
1665 be written to both destinations.
1666
1667 Returns the error code. */
1668 static int
1669 read_response_body (struct http_stat *hs, int sock, FILE *fp, wgint contlen,
1670 wgint contrange, bool chunked_transfer_encoding,
1671 char *url, char *warc_timestamp_str, char *warc_request_uuid,
1672 ip_address *warc_ip, char *type, int statcode, char *head)
1673 {
1674 int warc_payload_offset = 0;
1675 FILE *warc_tmp = NULL;
1676 int warcerr = 0;
1677 int flags = 0;
1678
1679 if (opt.warc_filename != NULL)
1680 {
1681 /* Open a temporary file where we can write the response before we
1682 add it to the WARC record. */
1683 warc_tmp = warc_tempfile ();
1684 if (warc_tmp == NULL)
1685 warcerr = WARC_TMP_FOPENERR;
1686
1687 if (warcerr == 0)
1688 {
1689 /* We should keep the response headers for the WARC record. */
1690 int head_len = strlen (head);
1691 int warc_tmp_written = fwrite (head, 1, head_len, warc_tmp);
1692 if (warc_tmp_written != head_len)
1693 warcerr = WARC_TMP_FWRITEERR;
1694 warc_payload_offset = head_len;
1695 }
1696
1697 if (warcerr != 0)
1698 {
1699 if (warc_tmp != NULL)
1700 fclose (warc_tmp);
1701 return warcerr;
1702 }
1703 }
1704
1705 if (fp != NULL)
1706 {
1707 /* This confuses the timestamping code that checks for file size.
1708 #### The timestamping code should be smarter about file size. */
1709 if (opt.save_headers && hs->restval == 0)
1710 fwrite (head, 1, strlen (head), fp);
1711 }
1712
1713 /* Read the response body. */
1714 if (contlen != -1)
1715 /* If content-length is present, read that much; otherwise, read
1716 until EOF. The HTTP spec doesn't require the server to
1717 actually close the connection when it's done sending data. */
1718 flags |= rb_read_exactly;
1719 if (fp != NULL && hs->restval > 0 && contrange == 0)
1720 /* If the server ignored our range request, instruct fd_read_body
1721 to skip the first RESTVAL bytes of body. */
1722 flags |= rb_skip_startpos;
1723 if (chunked_transfer_encoding)
1724 flags |= rb_chunked_transfer_encoding;
1725
1726 if (hs->remote_encoding == ENC_GZIP)
1727 flags |= rb_compressed_gzip;
1728
1729 hs->len = hs->restval;
1730 hs->rd_size = 0;
1731 /* Download the response body and write it to fp.
1732 If we are working on a WARC file, we simultaneously write the
1733 response body to warc_tmp. */
1734 hs->res = fd_read_body (hs->local_file, sock, fp, contlen != -1 ? contlen : 0,
1735 hs->restval, &hs->rd_size, &hs->len, &hs->dltime,
1736 flags, warc_tmp);
1737 if (hs->res >= 0)
1738 {
1739 if (warc_tmp != NULL)
1740 {
1741 /* Create a response record and write it to the WARC file.
1742 Note: per the WARC standard, the request and response should share
1743 the same date header. We re-use the timestamp of the request.
1744 The response record should also refer to the uuid of the request. */
1745 bool r = warc_write_response_record (url, warc_timestamp_str,
1746 warc_request_uuid, warc_ip,
1747 warc_tmp, warc_payload_offset,
1748 type, statcode, hs->newloc);
1749
1750 /* warc_write_response_record has closed warc_tmp. */
1751
1752 if (! r)
1753 return WARC_ERR;
1754 }
1755
1756 return RETRFINISHED;
1757 }
1758
1759 if (warc_tmp != NULL)
1760 fclose (warc_tmp);
1761
1762 if (hs->res == -2)
1763 {
1764 /* Error while writing to fd. */
1765 return FWRITEERR;
1766 }
1767 else if (hs->res == -3)
1768 {
1769 /* Error while writing to warc_tmp. */
1770 return WARC_TMP_FWRITEERR;
1771 }
1772 else
1773 {
1774 /* A read error! */
1775 xfree (hs->rderrmsg);
1776 hs->rderrmsg = xstrdup (fd_errstr (sock));
1777 return RETRFINISHED;
1778 }
1779 }
1780
1781 #define BEGINS_WITH(line, string_constant) \
1782 (!c_strncasecmp (line, string_constant, sizeof (string_constant) - 1) \
1783 && (c_isspace (line[sizeof (string_constant) - 1]) \
1784 || !line[sizeof (string_constant) - 1]))
1785
1786 #define SET_USER_AGENT(req) do { \
1787 if (!opt.useragent) \
1788 request_set_header (req, "User-Agent", \
1789 aprintf ("Wget/%s", \
1790 version_string), \
1791 rel_value); \
1792 else if (*opt.useragent) \
1793 request_set_header (req, "User-Agent", opt.useragent, rel_none); \
1794 } while (0)
1795
1796 /*
1797 Convert time_t to one of valid HTTP date formats
1798 ie. rfc1123-date.
1799
1800 HTTP-date = rfc1123-date | rfc850-date | asctime-date
1801 rfc1123-date = wkday "," SP date1 SP time SP "GMT"
1802 rfc850-date = weekday "," SP date2 SP time SP "GMT"
1803 asctime-date = wkday SP date3 SP time SP 4DIGIT
1804 date1 = 2DIGIT SP month SP 4DIGIT
1805 ; day month year (e.g., 02 Jun 1982)
1806 date2 = 2DIGIT "-" month "-" 2DIGIT
1807 ; day-month-year (e.g., 02-Jun-82)
1808 date3 = month SP ( 2DIGIT | ( SP 1DIGIT ))
1809 ; month day (e.g., Jun 2)
1810 time = 2DIGIT ":" 2DIGIT ":" 2DIGIT
1811 ; 00:00:00 - 23:59:59
1812 wkday = "Mon" | "Tue" | "Wed"
1813 | "Thu" | "Fri" | "Sat" | "Sun"
1814 weekday = "Monday" | "Tuesday" | "Wednesday"
1815 | "Thursday" | "Friday" | "Saturday" | "Sunday"
1816 month = "Jan" | "Feb" | "Mar" | "Apr"
1817 | "May" | "Jun" | "Jul" | "Aug"
1818 | "Sep" | "Oct" | "Nov" | "Dec"
1819
1820 source: RFC2616 */
1821 static uerr_t
1822 time_to_rfc1123 (time_t time, char *buf, size_t bufsize)
1823 {
1824 static const char *wkday[] = { "Sun", "Mon", "Tue", "Wed",
1825 "Thu", "Fri", "Sat" };
1826 static const char *month[] = { "Jan", "Feb", "Mar", "Apr",
1827 "May", "Jun", "Jul", "Aug",
1828 "Sep", "Oct", "Nov", "Dec" };
1829
1830 struct tm *gtm = gmtime (&time);
1831 if (!gtm)
1832 {
1833 logprintf (LOG_NOTQUIET,
1834 _("gmtime failed. This is probably a bug.\n"));
1835 return TIMECONV_ERR;
1836 }
1837
1838 /* rfc1123 example: Thu, 01 Jan 1998 22:12:57 GMT */
1839 snprintf (buf, bufsize, "%s, %02d %s %04d %02d:%02d:%02d GMT",
1840 wkday[gtm->tm_wday],
1841 gtm->tm_mday, month[gtm->tm_mon],
1842 gtm->tm_year + 1900, gtm->tm_hour,
1843 gtm->tm_min, gtm->tm_sec);
1844
1845 return RETROK;
1846 }
1847
1848 static struct request *
1849 initialize_request (const struct url *u, struct http_stat *hs, int *dt, struct url *proxy,
1850 bool inhibit_keep_alive, bool *basic_auth_finished,
1851 wgint *body_data_size, char **user, char **passwd, uerr_t *ret)
1852 {
1853 bool head_only = !!(*dt & HEAD_ONLY);
1854 struct request *req;
1855
1856 /* Prepare the request to send. */
1857 {
1858 char *meth_arg;
1859 const char *meth = "GET";
1860 if (head_only)
1861 meth = "HEAD";
1862 else if (opt.method)
1863 meth = opt.method;
1864 /* Use the full path, i.e. one that includes the leading slash and
1865 the query string. E.g. if u->path is "foo/bar" and u->query is
1866 "param=value", full_path will be "/foo/bar?param=value". */
1867 if (proxy
1868 #ifdef HAVE_SSL
1869 /* When using SSL over proxy, CONNECT establishes a direct
1870 connection to the HTTPS server. Therefore use the same
1871 argument as when talking to the server directly. */
1872 && u->scheme != SCHEME_HTTPS
1873 #endif
1874 )
1875 meth_arg = xstrdup (u->url);
1876 else
1877 meth_arg = url_full_path (u);
1878 req = request_new (meth, meth_arg);
1879 }
1880
1881 /* Generate the Host header, HOST:PORT. Take into account that:
1882
1883 - Broken server-side software often doesn't recognize the PORT
1884 argument, so we must generate "Host: www.server.com" instead of
1885 "Host: www.server.com:80" (and likewise for https port).
1886
1887 - IPv6 addresses contain ":", so "Host: 3ffe:8100:200:2::2:1234"
1888 becomes ambiguous and needs to be rewritten as "Host:
1889 [3ffe:8100:200:2::2]:1234". */
1890 {
1891 /* Formats arranged for hfmt[add_port][add_squares]. */
1892 static const char *hfmt[][2] = {
1893 { "%s", "[%s]" }, { "%s:%d", "[%s]:%d" }
1894 };
1895 int add_port = u->port != scheme_default_port (u->scheme);
1896 int add_squares = strchr (u->host, ':') != NULL;
1897 request_set_header (req, "Host",
1898 aprintf (hfmt[add_port][add_squares], u->host, u->port),
1899 rel_value);
1900 }
1901
1902 request_set_header (req, "Referer", hs->referer, rel_none);
1903 if (*dt & SEND_NOCACHE)
1904 {
1905 /* Cache-Control MUST be obeyed by all HTTP/1.1 caching mechanisms... */
1906 request_set_header (req, "Cache-Control", "no-cache", rel_none);
1907
1908 /* ... but some HTTP/1.0 caches doesn't implement Cache-Control. */
1909 request_set_header (req, "Pragma", "no-cache", rel_none);
1910 }
1911 if (*dt & IF_MODIFIED_SINCE)
1912 {
1913 char strtime[32];
1914 uerr_t err = time_to_rfc1123 (hs->orig_file_tstamp, strtime, countof (strtime));
1915
1916 if (err != RETROK)
1917 {
1918 logputs (LOG_VERBOSE, _("Cannot convert timestamp to http format. "
1919 "Falling back to time 0 as last modification "
1920 "time.\n"));
1921 strcpy (strtime, "Thu, 01 Jan 1970 00:00:00 GMT");
1922 }
1923 request_set_header (req, "If-Modified-Since", xstrdup (strtime), rel_value);
1924 }
1925 if (hs->restval)
1926 request_set_header (req, "Range",
1927 aprintf ("bytes=%s-",
1928 number_to_static_string (hs->restval)),
1929 rel_value);
1930 SET_USER_AGENT (req);
1931 request_set_header (req, "Accept", "*/*", rel_none);
1932 #ifdef HAVE_LIBZ
1933 if (opt.compression != compression_none)
1934 request_set_header (req, "Accept-Encoding", "gzip", rel_none);
1935 else
1936 #endif
1937 request_set_header (req, "Accept-Encoding", "identity", rel_none);
1938
1939 /* Find the username with priority */
1940 if (u->user)
1941 *user = u->user;
1942 else if (opt.user && (opt.use_askpass || opt.ask_passwd))
1943 *user = opt.user;
1944 else if (opt.http_user)
1945 *user = opt.http_user;
1946 else if (opt.user)
1947 *user = opt.user;
1948 else
1949 *user = NULL;
1950
1951 /* Find the password with priority */
1952 if (u->passwd)
1953 *passwd = u->passwd;
1954 else if (opt.passwd && (opt.use_askpass || opt.ask_passwd))
1955 *passwd = opt.passwd;
1956 else if (opt.http_passwd)
1957 *passwd = opt.http_passwd;
1958 else if (opt.passwd)
1959 *passwd = opt.passwd;
1960 else
1961 *passwd = NULL;
1962
1963 /* Check for ~/.netrc if none of the above match */
1964 if (opt.netrc && (!*user || !*passwd))
1965 search_netrc (u->host, (const char **) user, (const char **) passwd, 0, NULL);
1966
1967 /* We only do "site-wide" authentication with "global" user/password
1968 * values unless --auth-no-challenge has been requested; URL user/password
1969 * info overrides. */
1970 if (*user && *passwd && (!u->user || opt.auth_without_challenge))
1971 {
1972 /* If this is a host for which we've already received a Basic
1973 * challenge, we'll go ahead and send Basic authentication creds. */
1974 *basic_auth_finished = maybe_send_basic_creds (u->host, *user, *passwd, req);
1975 }
1976
1977 if (inhibit_keep_alive)
1978 request_set_header (req, "Connection", "Close", rel_none);
1979 else
1980 {
1981 request_set_header (req, "Connection", "Keep-Alive", rel_none);
1982 if (proxy)
1983 request_set_header (req, "Proxy-Connection", "Keep-Alive", rel_none);
1984 }
1985
1986 if (opt.method)
1987 {
1988
1989 if (opt.body_data || opt.body_file)
1990 {
1991 request_set_header (req, "Content-Type",
1992 "application/x-www-form-urlencoded", rel_none);
1993
1994 if (opt.body_data)
1995 *body_data_size = strlen (opt.body_data);
1996 else
1997 {
1998 *body_data_size = file_size (opt.body_file);
1999 if (*body_data_size == -1)
2000 {
2001 logprintf (LOG_NOTQUIET, _("BODY data file %s missing: %s\n"),
2002 quote (opt.body_file), strerror (errno));
2003 request_free (&req);
2004 *ret = FILEBADFILE;
2005 return NULL;
2006 }
2007 }
2008 request_set_header (req, "Content-Length",
2009 xstrdup (number_to_static_string (*body_data_size)),
2010 rel_value);
2011 }
2012 else if (c_strcasecmp (opt.method, "post") == 0
2013 || c_strcasecmp (opt.method, "put") == 0
2014 || c_strcasecmp (opt.method, "patch") == 0)
2015 request_set_header (req, "Content-Length", "0", rel_none);
2016 }
2017 return req;
2018 }
2019
2020 static void
2021 initialize_proxy_configuration (const struct url *u, struct request *req,
2022 struct url *proxy, char **proxyauth)
2023 {
2024 char *proxy_user, *proxy_passwd;
2025 /* For normal username and password, URL components override
2026 command-line/wgetrc parameters. With proxy
2027 authentication, it's the reverse, because proxy URLs are
2028 normally the "permanent" ones, so command-line args
2029 should take precedence. */
2030 if (opt.proxy_user && opt.proxy_passwd)
2031 {
2032 proxy_user = opt.proxy_user;
2033 proxy_passwd = opt.proxy_passwd;
2034 }
2035 else
2036 {
2037 proxy_user = proxy->user;
2038 proxy_passwd = proxy->passwd;
2039 }
2040 /* #### This does not appear right. Can't the proxy request,
2041 say, `Digest' authentication? */
2042 if (proxy_user && proxy_passwd)
2043 *proxyauth = basic_authentication_encode (proxy_user, proxy_passwd);
2044
2045 /* Proxy authorization over SSL is handled below. */
2046 #ifdef HAVE_SSL
2047 if (u->scheme != SCHEME_HTTPS)
2048 #endif
2049 request_set_header (req, "Proxy-Authorization", *proxyauth, rel_value);
2050 }
2051
2052 static uerr_t
2053 establish_connection (const struct url *u, const struct url **conn_ref,
2054 struct http_stat *hs, struct url *proxy,
2055 char **proxyauth,
2056 struct request **req_ref, bool *using_ssl,
2057 bool inhibit_keep_alive,
2058 int *sock_ref)
2059 {
2060 bool host_lookup_failed = false;
2061 int sock = *sock_ref;
2062 struct request *req = *req_ref;
2063 const struct url *conn = *conn_ref;
2064 struct response *resp;
2065 int write_error;
2066 int statcode;
2067
2068 if (! inhibit_keep_alive)
2069 {
2070 /* Look for a persistent connection to target host, unless a
2071 proxy is used. The exception is when SSL is in use, in which
2072 case the proxy is nothing but a passthrough to the target
2073 host, registered as a connection to the latter. */
2074 const struct url *relevant = conn;
2075 #ifdef HAVE_SSL
2076 if (u->scheme == SCHEME_HTTPS)
2077 relevant = u;
2078 #endif
2079
2080 if (persistent_available_p (relevant->host, relevant->port,
2081 #ifdef HAVE_SSL
2082 relevant->scheme == SCHEME_HTTPS,
2083 #else
2084 0,
2085 #endif
2086 &host_lookup_failed))
2087 {
2088 int family = socket_family (pconn.socket, ENDPOINT_PEER);
2089 sock = pconn.socket;
2090 *using_ssl = pconn.ssl;
2091 #if ENABLE_IPV6
2092 if (family == AF_INET6)
2093 logprintf (LOG_VERBOSE, _("Reusing existing connection to [%s]:%d.\n"),
2094 quotearg_style (escape_quoting_style, pconn.host),
2095 pconn.port);
2096 else
2097 #endif
2098 logprintf (LOG_VERBOSE, _("Reusing existing connection to %s:%d.\n"),
2099 quotearg_style (escape_quoting_style, pconn.host),
2100 pconn.port);
2101 DEBUGP (("Reusing fd %d.\n", sock));
2102 if (pconn.authorized)
2103 /* If the connection is already authorized, the "Basic"
2104 authorization added by code above is unnecessary and
2105 only hurts us. */
2106 request_remove_header (req, "Authorization");
2107 }
2108 else if (host_lookup_failed)
2109 {
2110 logprintf(LOG_NOTQUIET,
2111 _("%s: unable to resolve host address %s\n"),
2112 exec_name, quote (relevant->host));
2113 return HOSTERR;
2114 }
2115 else if (sock != -1)
2116 {
2117 sock = -1;
2118 }
2119 }
2120
2121 if (sock < 0)
2122 {
2123 sock = connect_to_host (conn->host, conn->port);
2124 if (sock == E_HOST)
2125 return HOSTERR;
2126 else if (sock < 0)
2127 return (retryable_socket_connect_error (errno)
2128 ? CONERROR : CONIMPOSSIBLE);
2129
2130 #ifdef HAVE_SSL
2131 if (proxy && u->scheme == SCHEME_HTTPS)
2132 {
2133 char *head;
2134 char *message;
2135 /* When requesting SSL URLs through proxies, use the
2136 CONNECT method to request passthrough. */
2137 struct request *connreq = request_new ("CONNECT",
2138 aprintf ("%s:%d", u->host, u->port));
2139 SET_USER_AGENT (connreq);
2140 if (proxyauth)
2141 {
2142 request_set_header (connreq, "Proxy-Authorization",
2143 *proxyauth, rel_value);
2144 /* Now that PROXYAUTH is part of the CONNECT request,
2145 zero it out so we don't send proxy authorization with
2146 the regular request below. */
2147 *proxyauth = NULL;
2148 }
2149 request_set_header (connreq, "Host",
2150 aprintf ("%s:%d", u->host, u->port),
2151 rel_value);
2152
2153 write_error = request_send (connreq, sock, 0);
2154 request_free (&connreq);
2155 if (write_error < 0)
2156 {
2157 CLOSE_INVALIDATE (sock);
2158 return WRITEFAILED;
2159 }
2160
2161 head = read_http_response_head (sock);
2162 if (!head)
2163 {
2164 logprintf (LOG_VERBOSE, _("Failed reading proxy response: %s\n"),
2165 fd_errstr (sock));
2166 CLOSE_INVALIDATE (sock);
2167 return HERR;
2168 }
2169 message = NULL;
2170 if (!*head)
2171 {
2172 xfree (head);
2173 goto failed_tunnel;
2174 }
2175 DEBUGP (("proxy responded with: [%s]\n", head));
2176
2177 resp = resp_new (head);
2178 statcode = resp_status (resp, &message);
2179 if (statcode < 0)
2180 {
2181 char *tms = datetime_str (time (NULL));
2182 logprintf (LOG_VERBOSE, "%d\n", statcode);
2183 logprintf (LOG_NOTQUIET, _("%s ERROR %d: %s.\n"), tms, statcode,
2184 quotearg_style (escape_quoting_style,
2185 _("Malformed status line")));
2186 xfree (head);
2187 return HERR;
2188 }
2189 xfree (hs->message);
2190 hs->message = xstrdup (message);
2191 resp_free (&resp);
2192 xfree (head);
2193 if (statcode != 200)
2194 {
2195 failed_tunnel:
2196 logprintf (LOG_NOTQUIET, _("Proxy tunneling failed: %s"),
2197 message ? quotearg_style (escape_quoting_style, message) : "?");
2198 xfree (message);
2199 return CONSSLERR;
2200 }
2201 xfree (message);
2202
2203 /* SOCK is now *really* connected to u->host, so update CONN
2204 to reflect this. That way register_persistent will
2205 register SOCK as being connected to u->host:u->port. */
2206 conn = u;
2207 }
2208
2209 if (conn->scheme == SCHEME_HTTPS)
2210 {
2211 if (!ssl_connect_wget (sock, u->host, NULL))
2212 {
2213 CLOSE_INVALIDATE (sock);
2214 return CONSSLERR;
2215 }
2216 else if (!ssl_check_certificate (sock, u->host))
2217 {
2218 CLOSE_INVALIDATE (sock);
2219 return VERIFCERTERR;
2220 }
2221 *using_ssl = true;
2222 }
2223 #endif /* HAVE_SSL */
2224 }
2225 *conn_ref = conn;
2226 *req_ref = req;
2227 *sock_ref = sock;
2228 return RETROK;
2229 }
2230
2231 static uerr_t
2232 set_file_timestamp (struct http_stat *hs)
2233 {
2234 bool local_dot_orig_file_exists = false;
2235 char *local_filename = NULL;
2236 struct stat st;
2237 char buf[1024];
2238
2239 if (opt.backup_converted)
2240 /* If -K is specified, we'll act on the assumption that it was specified
2241 last time these files were downloaded as well, and instead of just
2242 comparing local file X against server file X, we'll compare local
2243 file X.orig (if extant, else X) against server file X. If -K
2244 _wasn't_ specified last time, or the server contains files called
2245 *.orig, -N will be back to not operating correctly with -k. */
2246 {
2247 size_t filename_len = strlen (hs->local_file);
2248 char *filename_plus_orig_suffix;
2249
2250 if (filename_len + sizeof (ORIG_SFX) > sizeof (buf))
2251 filename_plus_orig_suffix = xmalloc (filename_len + sizeof (ORIG_SFX));
2252 else
2253 filename_plus_orig_suffix = buf;
2254
2255 /* Would a single s[n]printf() call be faster? --dan
2256
2257 Definitely not. sprintf() is horribly slow. It's a
2258 different question whether the difference between the two
2259 affects a program. Usually I'd say "no", but at one
2260 point I profiled Wget, and found that a measurable and
2261 non-negligible amount of time was lost calling sprintf()
2262 in url.c. Replacing sprintf with inline calls to
2263 strcpy() and number_to_string() made a difference.
2264 --hniksic */
2265 memcpy (filename_plus_orig_suffix, hs->local_file, filename_len);
2266 memcpy (filename_plus_orig_suffix + filename_len,
2267 ORIG_SFX, sizeof (ORIG_SFX));
2268
2269 /* Try to stat() the .orig file. */
2270 if (stat (filename_plus_orig_suffix, &st) == 0)
2271 {
2272 local_dot_orig_file_exists = true;
2273 local_filename = filename_plus_orig_suffix;
2274 }
2275 }
2276
2277 if (!local_dot_orig_file_exists)
2278 /* Couldn't stat() <file>.orig, so try to stat() <file>. */
2279 if (stat (hs->local_file, &st) == 0)
2280 {
2281 if (local_filename != buf)
2282 xfree (local_filename);
2283 local_filename = hs->local_file;
2284 }
2285
2286 if (local_filename != NULL)
2287 /* There was a local file, so we'll check later to see if the version
2288 the server has is the same version we already have, allowing us to
2289 skip a download. */
2290 {
2291 if (local_filename == buf || local_filename == hs->local_file)
2292 hs->orig_file_name = xstrdup (local_filename); // on stack or a copy, make a heap copy
2293 else
2294 hs->orig_file_name = local_filename; // was previously malloc'ed
2295 hs->orig_file_size = st.st_size;
2296 hs->orig_file_tstamp = st.st_mtime;
2297 #ifdef WINDOWS
2298 /* Modification time granularity is 2 seconds for Windows, so
2299 increase local time by 1 second for later comparison. */
2300 ++hs->orig_file_tstamp;
2301 #endif
2302 hs->timestamp_checked = true;
2303 }
2304
2305 return RETROK;
2306 }
2307
2308 static uerr_t
2309 check_file_output (const struct url *u, struct http_stat *hs,
2310 struct response *resp, char *hdrval, size_t hdrsize)
2311 {
2312 /* Determine the local filename if needed. Notice that if -O is used
2313 * hstat.local_file is set by http_loop to the argument of -O. */
2314 if (!hs->local_file)
2315 {
2316 char *local_file = NULL;
2317
2318 /* Honor Content-Disposition whether possible. */
2319 if (!opt.content_disposition
2320 || !resp_header_copy (resp, "Content-Disposition",
2321 hdrval, hdrsize)
2322 || !parse_content_disposition (hdrval, &local_file))
2323 {
2324 /* The Content-Disposition header is missing or broken.
2325 * Choose unique file name according to given URL. */
2326 hs->local_file = url_file_name (u, NULL);
2327 }
2328 else
2329 {
2330 DEBUGP (("Parsed filename from Content-Disposition: %s\n",
2331 local_file));
2332 hs->local_file = url_file_name (u, local_file);
2333 }
2334
2335 xfree (local_file);
2336 }
2337
2338 hs->temporary = opt.delete_after || opt.spider || !acceptable (hs->local_file);
2339 if (hs->temporary)
2340 {
2341 char *tmp = aprintf ("%s.tmp", hs->local_file);
2342 xfree (hs->local_file);
2343 hs->local_file = tmp;
2344 }
2345
2346 /* TODO: perform this check only once. */
2347 if (!hs->existence_checked && file_exists_p (hs->local_file, NULL))
2348 {
2349 if (opt.noclobber && !opt.output_document)
2350 {
2351 /* If opt.noclobber is turned on and file already exists, do not
2352 retrieve the file. But if the output_document was given, then this
2353 test was already done and the file didn't exist. Hence the !opt.output_document */
2354 return RETRUNNEEDED;
2355 }
2356 else if (!ALLOW_CLOBBER)
2357 {
2358 char *unique = unique_name_passthrough (hs->local_file);
2359 if (unique != hs->local_file)
2360 xfree (hs->local_file);
2361 hs->local_file = unique;
2362 }
2363 }
2364 hs->existence_checked = true;
2365
2366 /* Support timestamping */
2367 if (opt.timestamping && !hs->timestamp_checked)
2368 {
2369 uerr_t timestamp_err = set_file_timestamp (hs);
2370 if (timestamp_err != RETROK)
2371 return timestamp_err;
2372 }
2373 return RETROK;
2374 }
2375
2376 static uerr_t
2377 check_auth (const struct url *u, char *user, char *passwd, struct response *resp,
2378 struct request *req, bool *ntlm_seen_ref, bool *retry,
2379 bool *basic_auth_finished_ref, bool *auth_finished_ref)
2380 {
2381 uerr_t auth_err = RETROK;
2382 bool basic_auth_finished = *basic_auth_finished_ref;
2383 bool auth_finished = *auth_finished_ref;
2384 bool ntlm_seen = *ntlm_seen_ref;
2385 char buf[256], *tmp = NULL;
2386
2387 *retry = false;
2388
2389 if (!auth_finished && (user && passwd))
2390 {
2391 /* IIS sends multiple copies of WWW-Authenticate, one with
2392 the value "negotiate", and other(s) with data. Loop over
2393 all the occurrences and pick the one we recognize. */
2394 int wapos;
2395 const char *www_authenticate = NULL;
2396 const char *wabeg, *waend;
2397 const char *digest = NULL, *basic = NULL, *ntlm = NULL;
2398
2399 for (wapos = 0; !ntlm
2400 && (wapos = resp_header_locate (resp, "WWW-Authenticate", wapos,
2401 &wabeg, &waend)) != -1;
2402 ++wapos)
2403 {
2404 param_token name, value;
2405 size_t len = waend - wabeg;
2406
2407 if (tmp != buf)
2408 xfree (tmp);
2409
2410 if (len < sizeof (buf))
2411 tmp = buf;
2412 else
2413 tmp = xmalloc (len + 1);
2414
2415 memcpy (tmp, wabeg, len);
2416 tmp[len] = 0;
2417
2418 www_authenticate = tmp;
2419
2420 for (;!ntlm;)
2421 {
2422 /* extract the auth-scheme */
2423 while (c_isspace (*www_authenticate)) www_authenticate++;
2424 name.e = name.b = www_authenticate;
2425 while (*name.e && !c_isspace (*name.e)) name.e++;
2426
2427 if (name.b == name.e)
2428 break;
2429
2430 DEBUGP (("Auth scheme found '%.*s'\n", (int) (name.e - name.b), name.b));
2431
2432 if (known_authentication_scheme_p (name.b, name.e))
2433 {
2434 if (BEGINS_WITH (name.b, "NTLM"))
2435 {
2436 ntlm = name.b;
2437 break; /* this is the most secure challenge, stop here */
2438 }
2439 else if (!digest && BEGINS_WITH (name.b, "Digest"))
2440 digest = name.b;
2441 else if (!basic && BEGINS_WITH (name.b, "Basic"))
2442 basic = name.b;
2443 }
2444
2445 /* now advance over the auth-params */
2446 www_authenticate = name.e;
2447 DEBUGP (("Auth param list '%s'\n", www_authenticate));
2448 while (extract_param (&www_authenticate, &name, &value, ',', NULL) && name.b && value.b)
2449 {
2450 DEBUGP (("Auth param %.*s=%.*s\n",
2451 (int) (name.e - name.b), name.b, (int) (value.e - value.b), value.b));
2452 }
2453 }
2454 }
2455
2456 if (!basic && !digest && !ntlm)
2457 {
2458 /* If the authentication header is missing or
2459 unrecognized, there's no sense in retrying. */
2460 logputs (LOG_NOTQUIET, _("Unknown authentication scheme.\n"));
2461 }
2462 else if (!basic_auth_finished
2463 || !basic)
2464 {
2465 char *pth = url_full_path (u);
2466 const char *value;
2467 uerr_t *auth_stat;
2468 auth_stat = xmalloc (sizeof (uerr_t));
2469 *auth_stat = RETROK;
2470
2471 if (ntlm)
2472 www_authenticate = ntlm;
2473 else if (digest)
2474 www_authenticate = digest;
2475 else
2476 www_authenticate = basic;
2477
2478 logprintf (LOG_NOTQUIET, _("Authentication selected: %s\n"), www_authenticate);
2479
2480 value = create_authorization_line (www_authenticate,
2481 user, passwd,
2482 request_method (req),
2483 pth,
2484 &auth_finished,
2485 auth_stat);
2486
2487 auth_err = *auth_stat;
2488 xfree (auth_stat);
2489 xfree (pth);
2490 if (auth_err == RETROK)
2491 {
2492 request_set_header (req, "Authorization", value, rel_value);
2493
2494 if (BEGINS_WITH (www_authenticate, "NTLM"))
2495 ntlm_seen = true;
2496 else if (!u->user && BEGINS_WITH (www_authenticate, "Basic"))
2497 {
2498 /* Need to register this host as using basic auth,
2499 * so we automatically send creds next time. */
2500 register_basic_auth_host (u->host);
2501 }
2502
2503 *retry = true;
2504 goto cleanup;
2505 }
2506 else
2507 {
2508 /* Creating the Authorization header went wrong */
2509 xfree (value);
2510 }
2511 }
2512 else
2513 {
2514 /* We already did Basic auth, and it failed. Gotta
2515 * give up. */
2516 }
2517 }
2518
2519 cleanup:
2520 if (tmp != buf)
2521 xfree (tmp);
2522 *ntlm_seen_ref = ntlm_seen;
2523 *basic_auth_finished_ref = basic_auth_finished;
2524 *auth_finished_ref = auth_finished;
2525 return auth_err;
2526 }
2527
2528 static uerr_t
2529 open_output_stream (struct http_stat *hs, int count, FILE **fp)
2530 {
2531 /* 2005-06-17 SMS.
2532 For VMS, define common fopen() optional arguments.
2533 */
2534 #ifdef __VMS
2535 # define FOPEN_OPT_ARGS "fop=sqo", "acc", acc_cb, &open_id
2536 # define FOPEN_BIN_FLAG 3
2537 #else /* def __VMS */
2538 # define FOPEN_BIN_FLAG true
2539 #endif /* def __VMS [else] */
2540
2541 /* Open the local file. */
2542 if (!output_stream)
2543 {
2544 mkalldirs (hs->local_file);
2545 if (opt.backups)
2546 rotate_backups (hs->local_file);
2547 if (hs->restval)
2548 {
2549 #ifdef __VMS
2550 int open_id;
2551
2552 open_id = 21;
2553 *fp = fopen (hs->local_file, "ab", FOPEN_OPT_ARGS);
2554 #else /* def __VMS */
2555 *fp = fopen (hs->local_file, "ab");
2556 #endif /* def __VMS [else] */
2557 }
2558 else if (ALLOW_CLOBBER || count > 0)
2559 {
2560 if (opt.unlink_requested && file_exists_p (hs->local_file, NULL))
2561 {
2562 if (unlink (hs->local_file) < 0)
2563 {
2564 logprintf (LOG_NOTQUIET, "%s: %s\n", hs->local_file,
2565 strerror (errno));
2566 return UNLINKERR;
2567 }
2568 }
2569
2570 #ifdef __VMS
2571 int open_id;
2572
2573 open_id = 22;
2574 *fp = fopen (hs->local_file, "wb", FOPEN_OPT_ARGS);
2575 #else /* def __VMS */
2576 if (hs->temporary)
2577 {
2578 *fp = fdopen (open (hs->local_file, O_BINARY | O_CREAT | O_TRUNC | O_WRONLY, S_IRUSR | S_IWUSR), "wb");
2579 }
2580 else
2581 {
2582 *fp = fopen (hs->local_file, "wb");
2583 }
2584
2585 #endif /* def __VMS [else] */
2586 }
2587 else
2588 {
2589 *fp = fopen_excl (hs->local_file, FOPEN_BIN_FLAG);
2590 if (!*fp && errno == EEXIST)
2591 {
2592 /* We cannot just invent a new name and use it (which is
2593 what functions like unique_create typically do)
2594 because we told the user we'd use this name.
2595 Instead, return and retry the download. */
2596 logprintf (LOG_NOTQUIET,
2597 _("%s has sprung into existence.\n"),
2598 hs->local_file);
2599 return FOPEN_EXCL_ERR;
2600 }
2601 }
2602 if (!*fp)
2603 {
2604 logprintf (LOG_NOTQUIET, "%s: %s\n", hs->local_file, strerror (errno));
2605 return FOPENERR;
2606 }
2607 }
2608 else
2609 *fp = output_stream;
2610
2611 /* Print fetch message, if opt.verbose. */
2612 logprintf (LOG_VERBOSE, _("Saving to: %s\n"),
2613 HYPHENP (hs->local_file) ? quote ("STDOUT") : quote (hs->local_file));
2614
2615 return RETROK;
2616 }
2617
2618 /* Set proper type flags based on type string. */
2619 static void
2620 set_content_type (int *dt, const char *type)
2621 {
2622 /* If content-type is not given, assume text/html. This is because
2623 of the multitude of broken CGI's that "forget" to generate the
2624 content-type. */
2625 if (!type ||
2626 0 == c_strcasecmp (type, TEXTHTML_S) ||
2627 0 == c_strcasecmp (type, TEXTXHTML_S))
2628 *dt |= TEXTHTML;
2629 else
2630 *dt &= ~TEXTHTML;
2631
2632 if (type &&
2633 0 == c_strcasecmp (type, TEXTCSS_S))
2634 *dt |= TEXTCSS;
2635 else
2636 *dt &= ~TEXTCSS;
2637 }
2638
2639 #ifdef HAVE_METALINK
2640 /* Will return proper metalink_t structure if enough data was found in
2641 http response resp. Otherwise returns NULL.
2642 Two exit points: one for success and one for failure. */
2643 static metalink_t *
2644 metalink_from_http (const struct response *resp, const struct http_stat *hs,
2645 const struct url *u)
2646 {
2647 metalink_t *metalink = NULL;
2648 metalink_file_t *mfile = xnew0 (metalink_file_t);
2649 const char *val_beg, *val_end;
2650 int res_count = 0, meta_count = 0, hash_count = 0, sig_count = 0, i;
2651
2652 DEBUGP (("Checking for Metalink in HTTP response\n"));
2653
2654 /* Initialize metalink file for our simple use case. */
2655 if (hs->local_file)
2656 mfile->name = xstrdup (hs->local_file);
2657 else
2658 mfile->name = url_file_name (u, NULL);
2659
2660 /* Begin with 1-element array (for 0-termination). */
2661 mfile->checksums = xnew0 (metalink_checksum_t *);
2662 mfile->resources = xnew0 (metalink_resource_t *);
2663 mfile->metaurls = xnew0 (metalink_metaurl_t *);
2664
2665 /* Process the Content-Type header. */
2666 if (resp_header_locate (resp, "Content-Type", 0, &val_beg, &val_end) != -1)
2667 {
2668 metalink_metaurl_t murl = {0};
2669
2670 const char *type_beg, *type_end;
2671 char *typestr = NULL;
2672 char *namestr = NULL;
2673 size_t type_len;
2674
2675 DEBUGP (("Processing Content-Type header...\n"));
2676
2677 /* Find beginning of type. */
2678 type_beg = val_beg;
2679 while (type_beg < val_end && c_isspace (*type_beg))
2680 type_beg++;
2681
2682 /* Find end of type. */
2683 type_end = type_beg + 1;
2684 while (type_end < val_end &&
2685 *type_end != ';' &&
2686 *type_end != ' ' &&
2687 *type_end != '\r' &&
2688 *type_end != '\n')
2689 type_end++;
2690
2691 if (type_beg >= val_end || type_end > val_end)
2692 {
2693 DEBUGP (("Invalid Content-Type header. Ignoring.\n"));
2694 goto skip_content_type;
2695 }
2696
2697 type_len = type_end - type_beg;
2698 typestr = xstrndup (type_beg, type_len);
2699
2700 DEBUGP (("Content-Type: %s\n", typestr));
2701
2702 if (strcmp (typestr, "application/metalink4+xml"))
2703 {
2704 xfree (typestr);
2705 goto skip_content_type;
2706 }
2707
2708 /*
2709 Valid ranges for the "pri" attribute are from
2710 1 to 999999. Mirror servers with a lower value of the "pri"
2711 attribute have a higher priority, while mirrors with an undefined
2712 "pri" attribute are considered to have a value of 999999, which is
2713 the lowest priority.
2714
2715 rfc6249 section 3.1
2716 */
2717 murl.priority = DEFAULT_PRI;
2718
2719 murl.mediatype = typestr;
2720 typestr = NULL;
2721
2722 if (opt.content_disposition
2723 && resp_header_locate (resp, "Content-Disposition", 0, &val_beg, &val_end) != -1)
2724 {
2725 find_key_value (val_beg, val_end, "filename", &namestr);
2726 murl.name = namestr;
2727 namestr = NULL;
2728 }
2729
2730 murl.url = xstrdup (u->url);
2731
2732 DEBUGP (("URL=%s\n", murl.url));
2733 DEBUGP (("MEDIATYPE=%s\n", murl.mediatype));
2734 DEBUGP (("NAME=%s\n", murl.name ? murl.name : ""));
2735 DEBUGP (("PRIORITY=%d\n", murl.priority));
2736
2737 /* 1 slot from new resource, 1 slot for null-termination. */
2738 mfile->metaurls = xrealloc (mfile->metaurls,
2739 sizeof (metalink_metaurl_t *) * (meta_count + 2));
2740 mfile->metaurls[meta_count] = xnew0 (metalink_metaurl_t);
2741 *mfile->metaurls[meta_count] = murl;
2742 meta_count++;
2743 }
2744 skip_content_type:
2745
2746 /* Find all Link headers. */
2747 for (i = 0;
2748 (i = resp_header_locate (resp, "Link", i, &val_beg, &val_end)) != -1;
2749 i++)
2750 {
2751 char *rel = NULL, *reltype = NULL;
2752 char *urlstr = NULL;
2753 const char *url_beg, *url_end, *attrs_beg;
2754 size_t url_len;
2755
2756 /* Sample Metalink Link headers:
2757
2758 Link: <http://www2.example.com/dir1/dir2/dir3/dir4/dir5/example.ext>;
2759 rel=duplicate; pri=1; pref; geo=gb; depth=4
2760
2761 Link: <http://example.com/example.ext.asc>; rel=describedby;
2762 type="application/pgp-signature"
2763 */
2764
2765 /* Find beginning of URL. */
2766 url_beg = val_beg;
2767 while (url_beg < val_end - 1 && c_isspace (*url_beg))
2768 url_beg++;
2769
2770 /* Find end of URL. */
2771 /* The convention here is that end ptr points to one element after
2772 end of string. In this case, it should be pointing to the '>', which
2773 is one element after end of actual URL. Therefore, it should never point
2774 to val_end, which is one element after entire header value string. */
2775 url_end = url_beg + 1;
2776 while (url_end < val_end - 1 && *url_end != '>')
2777 url_end++;
2778
2779 if (url_beg >= val_end || url_end >= val_end ||
2780 *url_beg != '<' || *url_end != '>')
2781 {
2782 DEBUGP (("This is not a valid Link header. Ignoring.\n"));
2783 continue;
2784 }
2785
2786 /* Skip <. */
2787 url_beg++;
2788 url_len = url_end - url_beg;
2789
2790 /* URL found. Now handle the attributes. */
2791 attrs_beg = url_end + 1;
2792
2793 /* First we need to find out what type of link it is. Currently, we
2794 support rel=duplicate and rel=describedby. */
2795 if (!find_key_value (attrs_beg, val_end, "rel", &rel))
2796 {
2797 DEBUGP (("No rel value in Link header, skipping.\n"));
2798 continue;
2799 }
2800
2801 urlstr = xstrndup (url_beg, url_len);
2802 DEBUGP (("URL=%s\n", urlstr));
2803 DEBUGP (("rel=%s\n", rel));
2804
2805 if (!strcmp (rel, "describedby"))
2806 find_key_value (attrs_beg, val_end, "type", &reltype);
2807
2808 /* Handle signatures.
2809 Libmetalink only supports one signature per file. Therefore we stop
2810 as soon as we successfully get first supported signature. */
2811 if (sig_count == 0 &&
2812 reltype && !strcmp (reltype, "application/pgp-signature"))
2813 {
2814 /* Download the signature to a temporary file. */
2815 FILE *_output_stream = output_stream;
2816 bool _output_stream_regular = output_stream_regular;
2817
2818 output_stream = tmpfile ();
2819 if (output_stream)
2820 {
2821 struct iri *iri = iri_new ();
2822 struct url *url;
2823 int url_err;
2824
2825 set_uri_encoding (iri, opt.locale, true);
2826 url = url_parse (urlstr, &url_err, iri, false);
2827
2828 if (!url)
2829 {
2830 logprintf (LOG_NOTQUIET, _("When downloading signature:\n"
2831 "%s: %s.\n"), urlstr, url_error (url_err));
2832 iri_free (iri);
2833 }
2834 else
2835 {
2836 /* Avoid recursive Metalink from HTTP headers. */
2837 bool _metalink_http = opt.metalink_over_http;
2838 uerr_t retr_err;
2839
2840 opt.metalink_over_http = false;
2841 retr_err = retrieve_url (url, urlstr, NULL, NULL,
2842 NULL, NULL, false, iri, false);
2843 opt.metalink_over_http = _metalink_http;
2844
2845 url_free (url);
2846 iri_free (iri);
2847
2848 if (retr_err == RETROK)
2849 {
2850 /* Signature is in the temporary file. Read it into
2851 metalink resource structure. */
2852 metalink_signature_t msig;
2853 size_t siglen;
2854
2855 fseek (output_stream, 0, SEEK_END);
2856 siglen = ftell (output_stream);
2857 fseek (output_stream, 0, SEEK_SET);
2858
2859 DEBUGP (("siglen=%lu\n", siglen));
2860
2861 msig.signature = xmalloc (siglen + 1);
2862 if (fread (msig.signature, siglen, 1, output_stream) != 1)
2863 {
2864 logputs (LOG_NOTQUIET,
2865 _("Unable to read signature content from "
2866 "temporary file. Skipping.\n"));
2867 xfree (msig.signature);
2868 }
2869 else
2870 {
2871 msig.signature[siglen] = '\0'; /* Just in case. */
2872 msig.mediatype = xstrdup ("application/pgp-signature");
2873
2874 DEBUGP (("Signature (%s):\n%s\n",
2875 msig.mediatype, msig.signature));
2876
2877 mfile->signature = xnew (metalink_signature_t);
2878 *mfile->signature = msig;
2879
2880 sig_count++;
2881 }
2882 }
2883 }
2884 fclose (output_stream);
2885 }
2886 else
2887 {
2888 logputs (LOG_NOTQUIET, _("Could not create temporary file. "
2889 "Skipping signature download.\n"));
2890 }
2891 output_stream_regular = _output_stream_regular;
2892 output_stream = _output_stream;
2893 } /* Iterate over signatures. */
2894
2895 /* Handle Metalink resources. */
2896 else if (!strcmp (rel, "duplicate"))
2897 {
2898 metalink_resource_t mres = {0};
2899 char *pristr;
2900
2901 /*
2902 Valid ranges for the "pri" attribute are from
2903 1 to 999999. Mirror servers with a lower value of the "pri"
2904 attribute have a higher priority, while mirrors with an undefined
2905 "pri" attribute are considered to have a value of 999999, which is
2906 the lowest priority.
2907
2908 rfc6249 section 3.1
2909 */
2910 mres.priority = DEFAULT_PRI;
2911 if (find_key_value (url_end, val_end, "pri", &pristr))
2912 {
2913 long pri;
2914 char *end_pristr;
2915 /* Do not care for errno since 0 is error in this case. */
2916 pri = strtol (pristr, &end_pristr, 10);
2917 if (end_pristr != pristr + strlen (pristr) ||
2918 !VALID_PRI_RANGE (pri))
2919 {
2920 /* This is against the specification, so let's inform the user. */
2921 logprintf (LOG_NOTQUIET,
2922 _("Invalid pri value. Assuming %d.\n"),
2923 DEFAULT_PRI);
2924 }
2925 else
2926 mres.priority = pri;
2927 xfree (pristr);
2928 }
2929
2930 switch (url_scheme (urlstr))
2931 {
2932 case SCHEME_HTTP:
2933 mres.type = xstrdup ("http");
2934 break;
2935 #ifdef HAVE_SSL
2936 case SCHEME_HTTPS:
2937 mres.type = xstrdup ("https");
2938 break;
2939 case SCHEME_FTPS:
2940 mres.type = xstrdup ("ftps");
2941 break;
2942 #endif
2943 case SCHEME_FTP:
2944 mres.type = xstrdup ("ftp");
2945 break;
2946 default:
2947 DEBUGP (("Unsupported url scheme in %s. Skipping resource.\n", urlstr));
2948 }
2949
2950 if (mres.type)
2951 {
2952 DEBUGP (("TYPE=%s\n", mres.type));
2953
2954 /* At this point we have validated the new resource. */
2955
2956 find_key_value (url_end, val_end, "geo", &mres.location);
2957
2958 mres.url = urlstr;
2959 urlstr = NULL;
2960
2961 mres.preference = 0;
2962 if (has_key (url_end, val_end, "pref"))
2963 {
2964 DEBUGP (("This resource has preference\n"));
2965 mres.preference = 1;
2966 }
2967
2968 /* 1 slot from new resource, 1 slot for null-termination. */
2969 mfile->resources = xrealloc (mfile->resources,
2970 sizeof (metalink_resource_t *) * (res_count + 2));
2971 mfile->resources[res_count] = xnew0 (metalink_resource_t);
2972 *mfile->resources[res_count] = mres;
2973 res_count++;
2974 }
2975 } /* Handle resource link (rel=duplicate). */
2976
2977 /* Handle Metalink/XML resources. */
2978 else if (reltype && !strcmp (reltype, "application/metalink4+xml"))
2979 {
2980 metalink_metaurl_t murl = {0};
2981 char *pristr;
2982
2983 /*
2984 Valid ranges for the "pri" attribute are from
2985 1 to 999999. Mirror servers with a lower value of the "pri"
2986 attribute have a higher priority, while mirrors with an undefined
2987 "pri" attribute are considered to have a value of 999999, which is
2988 the lowest priority.
2989
2990 rfc6249 section 3.1
2991 */
2992 murl.priority = DEFAULT_PRI;
2993 if (find_key_value (url_end, val_end, "pri", &pristr))
2994 {
2995 long pri;
2996 char *end_pristr;
2997 /* Do not care for errno since 0 is error in this case. */
2998 pri = strtol (pristr, &end_pristr, 10);
2999 if (end_pristr != pristr + strlen (pristr) ||
3000 !VALID_PRI_RANGE (pri))
3001 {
3002 /* This is against the specification, so let's inform the user. */
3003 logprintf (LOG_NOTQUIET,
3004 _("Invalid pri value. Assuming %d.\n"),
3005 DEFAULT_PRI);
3006 }
3007 else
3008 murl.priority = pri;
3009 xfree (pristr);
3010 }
3011
3012 murl.mediatype = xstrdup (reltype);
3013
3014 DEBUGP (("MEDIATYPE=%s\n", murl.mediatype));
3015
3016 /* At this point we have validated the new resource. */
3017
3018 find_key_value (url_end, val_end, "name", &murl.name);
3019
3020 murl.url = urlstr;
3021 urlstr = NULL;
3022
3023 /* 1 slot from new resource, 1 slot for null-termination. */
3024 mfile->metaurls = xrealloc (mfile->metaurls,
3025 sizeof (metalink_metaurl_t *) * (meta_count + 2));
3026 mfile->metaurls[meta_count] = xnew0 (metalink_metaurl_t);
3027 *mfile->metaurls[meta_count] = murl;
3028 meta_count++;
3029 } /* Handle resource link (rel=describedby). */
3030 else
3031 DEBUGP (("This link header was not used for Metalink\n"));
3032
3033 xfree (urlstr);
3034 xfree (reltype);
3035 xfree (rel);
3036 } /* Iterate over link headers. */
3037
3038 /* Null-terminate resources array. */
3039 mfile->resources[res_count] = 0;
3040 mfile->metaurls[meta_count] = 0;
3041
3042 if (res_count == 0 && meta_count == 0)
3043 {
3044 DEBUGP (("No valid metalink references found.\n"));
3045 goto fail;
3046 }
3047
3048 /* Find all Digest headers. */
3049 for (i = 0;
3050 (i = resp_header_locate (resp, "Digest", i, &val_beg, &val_end)) != -1;
3051 i++)
3052 {
3053 const char *dig_pos;
3054 char *dig_type, *dig_hash;
3055
3056 /* Each Digest header can include multiple hashes. Example:
3057 Digest: SHA=thvDyvhfIqlvFe+A9MYgxAfm1q5=,unixsum=30637
3058 Digest: md5=HUXZLQLMuI/KZ5KDcJPcOA==
3059 */
3060 for (dig_pos = val_beg;
3061 (dig_pos = find_key_values (dig_pos, val_end, &dig_type, &dig_hash));
3062 dig_pos++)
3063 {
3064 /* The hash here is assumed to be base64. We need the hash in hex.
3065 Therefore we convert: base64 -> binary -> hex. */
3066 const size_t dig_hash_str_len = strlen (dig_hash);
3067 char bin_hash[256];
3068 ssize_t hash_bin_len;
3069
3070 // there is no hash with that size
3071 if (dig_hash_str_len >= sizeof (bin_hash))
3072 {
3073 DEBUGP (("Hash too long, ignored.\n"));
3074 xfree (dig_type);
3075 xfree (dig_hash);
3076 continue;
3077 }
3078
3079 hash_bin_len = wget_base64_decode (dig_hash, bin_hash, dig_hash_str_len * 3 / 4 + 1);
3080
3081 /* Detect malformed base64 input. */
3082 if (hash_bin_len < 0)
3083 {
3084 DEBUGP (("Malformed base64 input, ignored.\n"));
3085 xfree (dig_type);
3086 xfree (dig_hash);
3087 continue;
3088 }
3089
3090 /* One slot for me, one for zero-termination. */
3091 mfile->checksums =
3092 xrealloc (mfile->checksums,
3093 sizeof (metalink_checksum_t *) * (hash_count + 2));
3094 mfile->checksums[hash_count] = xnew (metalink_checksum_t);
3095 mfile->checksums[hash_count]->type = dig_type;
3096
3097 mfile->checksums[hash_count]->hash = xmalloc ((size_t)hash_bin_len * 2 + 1);
3098 wg_hex_to_string (mfile->checksums[hash_count]->hash, bin_hash, (size_t)hash_bin_len);
3099
3100 xfree (dig_hash);
3101
3102 hash_count++;
3103 }
3104 }
3105
3106 /* Zero-terminate checksums array. */
3107 mfile->checksums[hash_count] = 0;
3108
3109 /*
3110 If Instance Digests are not provided by the Metalink servers, the
3111 Link header fields pertaining to this specification MUST be ignored.
3112
3113 rfc6249 section 6
3114 */
3115 if (res_count && hash_count == 0)
3116 {
3117 logputs (LOG_VERBOSE,
3118 _("Could not find acceptable digest for Metalink resources.\n"
3119 "Ignoring them.\n"));
3120 goto fail;
3121 }
3122
3123 /* Metalink data is OK. Now we just need to sort the resources based
3124 on their priorities, preference, and perhaps location. */
3125 stable_sort (mfile->resources, res_count, sizeof (metalink_resource_t *), metalink_res_cmp);
3126 stable_sort (mfile->metaurls, meta_count, sizeof (metalink_metaurl_t *), metalink_meta_cmp);
3127
3128 /* Restore sensible preference values (in case someone cares to look). */
3129 for (i = 0; i < res_count; ++i)
3130 mfile->resources[i]->preference = 1000000 - mfile->resources[i]->priority;
3131
3132 metalink = xnew0 (metalink_t);
3133 metalink->files = xmalloc (sizeof (metalink_file_t *) * 2);
3134 metalink->files[0] = mfile;
3135 metalink->files[1] = 0;
3136 metalink->origin = xstrdup (u->url);
3137 metalink->version = METALINK_VERSION_4;
3138 /* Leave other fields set to 0. */
3139
3140 return metalink;
3141
3142 fail:
3143 /* Free all allocated memory. */
3144 if (metalink)
3145 metalink_delete (metalink);
3146 else
3147 metalink_file_delete (mfile);
3148 return NULL;
3149 }
3150 #endif /* HAVE_METALINK */
3151
3152 /* Retrieve a document through HTTP protocol. It recognizes status
3153 code, and correctly handles redirections. It closes the network
3154 socket. If it receives an error from the functions below it, it
3155 will print it if there is enough information to do so (almost
3156 always), returning the error to the caller (i.e. http_loop).
3157
3158 Various HTTP parameters are stored to hs.
3159
3160 If PROXY is non-NULL, the connection will be made to the proxy
3161 server, and u->url will be requested. */
3162 static uerr_t
3163 gethttp (const struct url *u, struct url *original_url, struct http_stat *hs,
3164 int *dt, struct url *proxy, struct iri *iri, int count)
3165 {
3166 struct request *req = NULL;
3167
3168 char *type = NULL;
3169 char *user, *passwd;
3170 char *proxyauth;
3171 int statcode;
3172 int write_error;
3173 wgint contlen, contrange;
3174 const struct url *conn;
3175 FILE *fp;
3176 int err;
3177 uerr_t retval;
3178 #ifdef HAVE_HSTS
3179 #ifdef TESTING
3180 /* we don't link against main.o when we're testing */
3181 hsts_store_t hsts_store = NULL;
3182 #else
3183 extern hsts_store_t hsts_store;
3184 #endif
3185 #endif
3186
3187 int sock = -1;
3188
3189 /* Set to 1 when the authorization has already been sent and should
3190 not be tried again. */
3191 bool auth_finished = false;
3192
3193 /* Set to 1 when just globally-set Basic authorization has been sent;
3194 * should prevent further Basic negotiations, but not other
3195 * mechanisms. */
3196 bool basic_auth_finished = false;
3197
3198 /* Whether NTLM authentication is used for this request. */
3199 bool ntlm_seen = false;
3200
3201 /* Whether our connection to the remote host is through SSL. */
3202 bool using_ssl = false;
3203
3204 /* Whether a HEAD request will be issued (as opposed to GET or
3205 POST). */
3206 bool head_only = !!(*dt & HEAD_ONLY);
3207
3208 /* Whether conditional get request will be issued. */
3209 bool cond_get = !!(*dt & IF_MODIFIED_SINCE);
3210
3211 #ifdef HAVE_METALINK
3212 /* Are we looking for metalink info in HTTP headers? */
3213 bool metalink = !!(*dt & METALINK_METADATA);
3214 #endif
3215
3216 char *head = NULL;
3217 struct response *resp = NULL;
3218 char hdrval[512];
3219 char *message = NULL;
3220
3221 /* Declare WARC variables. */
3222 bool warc_enabled = (opt.warc_filename != NULL);
3223 FILE *warc_tmp = NULL;
3224 char warc_timestamp_str [21];
3225 char warc_request_uuid [48];
3226 ip_address warc_ip_buf, *warc_ip = NULL;
3227 off_t warc_payload_offset = -1;
3228
3229 /* Whether this connection will be kept alive after the HTTP request
3230 is done. */
3231 bool keep_alive;
3232
3233 /* Is the server using the chunked transfer encoding? */
3234 bool chunked_transfer_encoding = false;
3235
3236 /* Whether keep-alive should be inhibited. */
3237 bool inhibit_keep_alive =
3238 !opt.http_keep_alive || opt.ignore_length;
3239
3240 /* Headers sent when using POST. */
3241 wgint body_data_size = 0;
3242
3243 #ifdef HAVE_SSL
3244 if (u->scheme == SCHEME_HTTPS)
3245 {
3246 /* Initialize the SSL context. After this has once been done,
3247 it becomes a no-op. */
3248 if (!ssl_init ())
3249 {
3250 scheme_disable (SCHEME_HTTPS);
3251 logprintf (LOG_NOTQUIET,
3252 _("Disabling SSL due to encountered errors.\n"));
3253 retval = SSLINITFAILED;
3254 goto cleanup;
3255 }
3256 }
3257 #endif /* HAVE_SSL */
3258
3259 /* Initialize certain elements of struct http_stat.
3260 * Since this function is called in a loop, we have to xfree certain
3261 * members. */
3262 hs->len = 0;
3263 hs->contlen = -1;
3264 hs->res = -1;
3265 xfree (hs->rderrmsg);
3266 xfree (hs->newloc);
3267 xfree (hs->remote_time);
3268 xfree (hs->error);
3269 xfree (hs->message);
3270 hs->local_encoding = ENC_NONE;
3271 hs->remote_encoding = ENC_NONE;
3272
3273 conn = u;
3274
3275 {
3276 uerr_t ret;
3277 req = initialize_request (u, hs, dt, proxy, inhibit_keep_alive,
3278 &basic_auth_finished, &body_data_size,
3279 &user, &passwd, &ret);
3280 if (req == NULL)
3281 {
3282 retval = ret;
3283 goto cleanup;
3284 }
3285 }
3286 retry_with_auth:
3287 /* We need to come back here when the initial attempt to retrieve
3288 without authorization header fails. (Expected to happen at least
3289 for the Digest authorization scheme.) */
3290
3291 if (opt.cookies)
3292 request_set_header (req, "Cookie",
3293 cookie_header (wget_cookie_jar,
3294 u->host, u->port, u->path,
3295 #ifdef HAVE_SSL
3296 u->scheme == SCHEME_HTTPS
3297 #else
3298 0
3299 #endif
3300 ),
3301 rel_value);
3302
3303 /* Add the user headers. */
3304 if (opt.user_headers)
3305 {
3306 int i;
3307 for (i = 0; opt.user_headers[i]; i++)
3308 request_set_user_header (req, opt.user_headers[i]);
3309 }
3310
3311 proxyauth = NULL;
3312 if (proxy)
3313 {
3314 conn = proxy;
3315 initialize_proxy_configuration (u, req, proxy, &proxyauth);
3316 }
3317 keep_alive = true;
3318
3319 /* Establish the connection. */
3320 if (inhibit_keep_alive)
3321 keep_alive = false;
3322
3323 {
3324 uerr_t conn_err = establish_connection (u, &conn, hs, proxy, &proxyauth, &req,
3325 &using_ssl, inhibit_keep_alive, &sock);
3326 if (conn_err != RETROK)
3327 {
3328 retval = conn_err;
3329 goto cleanup;
3330 }
3331 }
3332
3333 /* Open the temporary file where we will write the request. */
3334 if (warc_enabled)
3335 {
3336 warc_tmp = warc_tempfile ();
3337 if (warc_tmp == NULL)
3338 {
3339 CLOSE_INVALIDATE (sock);
3340 retval = WARC_TMP_FOPENERR;
3341 goto cleanup;
3342 }
3343
3344 if (! proxy)
3345 {
3346 warc_ip = &warc_ip_buf;
3347 socket_ip_address (sock, warc_ip, ENDPOINT_PEER);
3348 }
3349 }
3350
3351 /* Send the request to server. */
3352 write_error = request_send (req, sock, warc_tmp);
3353
3354 if (write_error >= 0)
3355 {
3356 if (opt.body_data)
3357 {
3358 DEBUGP (("[BODY data: %s]\n", opt.body_data));
3359 write_error = fd_write (sock, opt.body_data, body_data_size, -1);
3360 if (write_error >= 0 && warc_tmp != NULL)
3361 {
3362 int warc_tmp_written;
3363
3364 /* Remember end of headers / start of payload. */
3365 warc_payload_offset = ftello (warc_tmp);
3366
3367 /* Write a copy of the data to the WARC record. */
3368 warc_tmp_written = fwrite (opt.body_data, 1, body_data_size, warc_tmp);
3369 if (warc_tmp_written != body_data_size)
3370 write_error = -2;
3371 }
3372 }
3373 else if (opt.body_file && body_data_size != 0)
3374 {
3375 if (warc_tmp != NULL)
3376 /* Remember end of headers / start of payload */
3377 warc_payload_offset = ftello (warc_tmp);
3378
3379 write_error = body_file_send (sock, opt.body_file, body_data_size, warc_tmp);
3380 }
3381 }
3382
3383 if (write_error < 0)
3384 {
3385 CLOSE_INVALIDATE (sock);
3386
3387 if (warc_tmp != NULL)
3388 fclose (warc_tmp);
3389
3390 if (write_error == -2)
3391 retval = WARC_TMP_FWRITEERR;
3392 else
3393 retval = WRITEFAILED;
3394 goto cleanup;
3395 }
3396 logprintf (LOG_VERBOSE, _("%s request sent, awaiting response... "),
3397 proxy ? "Proxy" : "HTTP");
3398 contlen = -1;
3399 contrange = 0;
3400 *dt &= ~RETROKF;
3401
3402
3403 if (warc_enabled)
3404 {
3405 bool warc_result;
3406
3407 /* Generate a timestamp and uuid for this request. */
3408 warc_timestamp (warc_timestamp_str, sizeof (warc_timestamp_str));
3409 warc_uuid_str (warc_request_uuid, sizeof (warc_request_uuid));
3410
3411 /* Create a request record and store it in the WARC file. */
3412 warc_result = warc_write_request_record (u->url, warc_timestamp_str,
3413 warc_request_uuid, warc_ip,
3414 warc_tmp, warc_payload_offset);
3415 if (! warc_result)
3416 {
3417 CLOSE_INVALIDATE (sock);
3418 retval = WARC_ERR;
3419 goto cleanup;
3420 }
3421
3422 /* warc_write_request_record has also closed warc_tmp. */
3423 }
3424
3425 /* Repeat while we receive a 10x response code. */
3426 {
3427 bool _repeat;
3428
3429 do
3430 {
3431 head = read_http_response_head (sock);
3432 if (!head)
3433 {
3434 if (errno == 0)
3435 {
3436 logputs (LOG_NOTQUIET, _("No data received.\n"));
3437 CLOSE_INVALIDATE (sock);
3438 retval = HEOF;
3439 }
3440 else
3441 {
3442 logprintf (LOG_NOTQUIET, _("Read error (%s) in headers.\n"),
3443 fd_errstr (sock));
3444 CLOSE_INVALIDATE (sock);
3445 retval = HERR;
3446 }
3447 goto cleanup;
3448 }
3449 DEBUGP (("\n---response begin---\n%s---response end---\n", head));
3450
3451 resp = resp_new (head);
3452
3453 /* Check for status line. */
3454 xfree (message);
3455 statcode = resp_status (resp, &message);
3456 if (statcode < 0)
3457 {
3458 char *tms = datetime_str (time (NULL));
3459 logprintf (LOG_VERBOSE, "%d\n", statcode);
3460 logprintf (LOG_NOTQUIET, _("%s ERROR %d: %s.\n"), tms, statcode,
3461 quotearg_style (escape_quoting_style,
3462 _("Malformed status line")));
3463 CLOSE_INVALIDATE (sock);
3464 retval = HERR;
3465 goto cleanup;
3466 }
3467
3468 if (H_10X (statcode))
3469 {
3470 xfree (head);
3471 resp_free (&resp);
3472 _repeat = true;
3473 DEBUGP (("Ignoring response\n"));
3474 }
3475 else
3476 {
3477 _repeat = false;
3478 }
3479 }
3480 while (_repeat);
3481 }
3482
3483 xfree (hs->message);
3484 hs->message = xstrdup (message);
3485 if (!opt.server_response)
3486 logprintf (LOG_VERBOSE, "%2d %s\n", statcode,
3487 message ? quotearg_style (escape_quoting_style, message) : "");
3488 else
3489 {
3490 logprintf (LOG_VERBOSE, "\n");
3491 print_server_response (resp, " ");
3492 }
3493
3494 if (!opt.ignore_length
3495 && resp_header_copy (resp, "Content-Length", hdrval, sizeof (hdrval)))
3496 {
3497 wgint parsed;
3498 errno = 0;
3499 parsed = str_to_wgint (hdrval, NULL, 10);
3500 if (parsed == WGINT_MAX && errno == ERANGE)
3501 {
3502 /* Out of range.
3503 #### If Content-Length is out of range, it most likely
3504 means that the file is larger than 2G and that we're
3505 compiled without LFS. In that case we should probably
3506 refuse to even attempt to download the file. */
3507 contlen = -1;
3508 }
3509 else if (parsed < 0)
3510 {
3511 /* Negative Content-Length; nonsensical, so we can't
3512 assume any information about the content to receive. */
3513 contlen = -1;
3514 }
3515 else
3516 contlen = parsed;
3517 }
3518
3519 /* Check for keep-alive related responses. */
3520 if (!inhibit_keep_alive)
3521 {
3522 if (resp_header_copy (resp, "Connection", hdrval, sizeof (hdrval)))
3523 {
3524 if (0 == c_strcasecmp (hdrval, "Close"))
3525 keep_alive = false;
3526 }
3527 }
3528
3529 chunked_transfer_encoding = false;
3530 if (resp_header_copy (resp, "Transfer-Encoding", hdrval, sizeof (hdrval))
3531 && 0 == c_strcasecmp (hdrval, "chunked"))
3532 chunked_transfer_encoding = true;
3533
3534 /* Handle (possibly multiple instances of) the Set-Cookie header. */
3535 if (opt.cookies)
3536 {
3537 int scpos;
3538 const char *scbeg, *scend;
3539 /* The jar should have been created by now. */
3540 assert (wget_cookie_jar != NULL);
3541 for (scpos = 0;
3542 (scpos = resp_header_locate (resp, "Set-Cookie", scpos,
3543 &scbeg, &scend)) != -1;
3544 ++scpos)
3545 {
3546 char buf[1024], *set_cookie;
3547 size_t len = scend - scbeg;
3548
3549 if (len < sizeof (buf))
3550 set_cookie = buf;
3551 else
3552 set_cookie = xmalloc (len + 1);
3553
3554 memcpy (set_cookie, scbeg, len);
3555 set_cookie[len] = 0;
3556
3557 cookie_handle_set_cookie (wget_cookie_jar, u->host, u->port,
3558 u->path, set_cookie);
3559
3560 if (set_cookie != buf)
3561 xfree (set_cookie);
3562 }
3563 }
3564
3565 if (keep_alive)
3566 /* The server has promised that it will not close the connection
3567 when we're done. This means that we can register it. */
3568 register_persistent (conn->host, conn->port, sock, using_ssl);
3569
3570 #ifdef HAVE_METALINK
3571 /* We need to check for the Metalink data in the very first response
3572 we get from the server (before redirections, authorization, etc.). */
3573 if (metalink)
3574 {
3575 hs->metalink = metalink_from_http (resp, hs, u);
3576 /* Bugfix: hs->local_file is NULL (opt.content_disposition). */
3577 if (!hs->local_file && hs->metalink && hs->metalink->origin)
3578 hs->local_file = xstrdup (hs->metalink->origin);
3579 xfree (hs->message);
3580 retval = RETR_WITH_METALINK;
3581 CLOSE_FINISH (sock);
3582 goto cleanup;
3583 }
3584 #endif
3585
3586 if (statcode == HTTP_STATUS_UNAUTHORIZED)
3587 {
3588 /* Authorization is required. */
3589 uerr_t auth_err = RETROK;
3590 bool retry;
3591 /* Normally we are not interested in the response body.
3592 But if we are writing a WARC file we are: we like to keep everything. */
3593 if (warc_enabled)
3594 {
3595 int _err;
3596 type = resp_header_strdup (resp, "Content-Type");
3597 _err = read_response_body (hs, sock, NULL, contlen, 0,
3598 chunked_transfer_encoding,
3599 u->url, warc_timestamp_str,
3600 warc_request_uuid, warc_ip, type,
3601 statcode, head);
3602 xfree (type);
3603
3604 if (_err != RETRFINISHED || hs->res < 0)
3605 {
3606 CLOSE_INVALIDATE (sock);
3607 retval = _err;
3608 goto cleanup;
3609 }
3610 else
3611 CLOSE_FINISH (sock);
3612 }
3613 else
3614 {
3615 /* Since WARC is disabled, we are not interested in the response body. */
3616 if (keep_alive && !head_only
3617 && skip_short_body (sock, contlen, chunked_transfer_encoding))
3618 CLOSE_FINISH (sock);
3619 else
3620 CLOSE_INVALIDATE (sock);
3621 }
3622
3623 pconn.authorized = false;
3624
3625 {
3626 auth_err = check_auth (u, user, passwd, resp, req,
3627 &ntlm_seen, &retry,
3628 &basic_auth_finished,
3629 &auth_finished);
3630 if (auth_err == RETROK && retry)
3631 {
3632 resp_free (&resp);
3633 xfree (message);
3634 xfree (head);
3635 goto retry_with_auth;
3636 }
3637 }
3638 if (auth_err == RETROK)
3639 retval = AUTHFAILED;
3640 else
3641 retval = auth_err;
3642 goto cleanup;
3643 }
3644 else /* statcode != HTTP_STATUS_UNAUTHORIZED */
3645 {
3646 /* Kludge: if NTLM is used, mark the TCP connection as authorized. */
3647 if (ntlm_seen)
3648 pconn.authorized = true;
3649 }
3650
3651 {
3652 uerr_t ret = check_file_output (u, hs, resp, hdrval, sizeof hdrval);
3653 if (ret != RETROK)
3654 {
3655 retval = ret;
3656 goto cleanup;
3657 }
3658 }
3659
3660 hs->statcode = statcode;
3661 xfree (hs->error);
3662 if (statcode == -1)
3663 hs->error = xstrdup (_("Malformed status line"));
3664 else if (!message || !*message)
3665 hs->error = xstrdup (_("(no description)"));
3666 else
3667 hs->error = xstrdup (message);
3668
3669 #ifdef HAVE_HSTS
3670 if (opt.hsts && hsts_store)
3671 {
3672 int64_t max_age;
3673 const char *hsts_params = resp_header_strdup (resp, "Strict-Transport-Security");
3674 bool include_subdomains;
3675
3676 if (parse_strict_transport_security (hsts_params, &max_age, &include_subdomains))
3677 {
3678 /* process strict transport security */
3679 if (hsts_store_entry (hsts_store, u->scheme, u->host, u->port, max_age, include_subdomains))
3680 DEBUGP(("Added new HSTS host: %s:%" PRIu32 " (max-age: %" PRId64 ", includeSubdomains: %s)\n",
3681 u->host,
3682 (uint32_t) u->port,
3683 max_age,
3684 (include_subdomains ? "true" : "false")));
3685 else
3686 DEBUGP(("Updated HSTS host: %s:%" PRIu32 " (max-age: %" PRId64 ", includeSubdomains: %s)\n",
3687 u->host,
3688 (uint32_t) u->port,
3689 max_age,
3690 (include_subdomains ? "true" : "false")));
3691 }
3692 xfree (hsts_params);
3693 }
3694 #endif
3695
3696 type = resp_header_strdup (resp, "Content-Type");
3697 if (type)
3698 {
3699 char *tmp = strchr (type, ';');
3700 if (tmp)
3701 {
3702 #ifdef ENABLE_IRI
3703 /* sXXXav: only needed if IRI support is enabled */
3704 char *tmp2 = tmp + 1;
3705 #endif
3706
3707 while (tmp > type && c_isspace (tmp[-1]))
3708 --tmp;
3709 *tmp = '\0';
3710
3711 #ifdef ENABLE_IRI
3712 /* Try to get remote encoding if needed */
3713 if (opt.enable_iri && !opt.encoding_remote)
3714 {
3715 tmp = parse_charset (tmp2);
3716 if (tmp)
3717 set_content_encoding (iri, tmp);
3718 xfree (tmp);
3719 }
3720 #endif
3721 }
3722 }
3723 xfree (hs->newloc);
3724 hs->newloc = resp_header_strdup (resp, "Location");
3725 xfree (hs->remote_time);
3726 hs->remote_time = resp_header_strdup (resp, "Last-Modified");
3727 if (!hs->remote_time) // now look for the Wayback Machine's timestamp
3728 hs->remote_time = resp_header_strdup (resp, "X-Archive-Orig-last-modified");
3729
3730 if (resp_header_copy (resp, "Content-Range", hdrval, sizeof (hdrval)))
3731 {
3732 wgint first_byte_pos, last_byte_pos, entity_length;
3733 if (parse_content_range (hdrval, &first_byte_pos, &last_byte_pos,
3734 &entity_length))
3735 {
3736 contrange = first_byte_pos;
3737 contlen = last_byte_pos - first_byte_pos + 1;
3738 }
3739 }
3740
3741 if (resp_header_copy (resp, "Content-Encoding", hdrval, sizeof (hdrval)))
3742 {
3743 hs->local_encoding = ENC_INVALID;
3744
3745 switch (hdrval[0])
3746 {
3747 case 'b': case 'B':
3748 if (0 == c_strcasecmp(hdrval, "br"))
3749 hs->local_encoding = ENC_BROTLI;
3750 break;
3751 case 'c': case 'C':
3752 if (0 == c_strcasecmp(hdrval, "compress"))
3753 hs->local_encoding = ENC_COMPRESS;
3754 break;
3755 case 'd': case 'D':
3756 if (0 == c_strcasecmp(hdrval, "deflate"))
3757 hs->local_encoding = ENC_DEFLATE;
3758 break;
3759 case 'g': case 'G':
3760 if (0 == c_strcasecmp(hdrval, "gzip"))
3761 hs->local_encoding = ENC_GZIP;
3762 break;
3763 case 'i': case 'I':
3764 if (0 == c_strcasecmp(hdrval, "identity"))
3765 hs->local_encoding = ENC_NONE;
3766 break;
3767 case 'x': case 'X':
3768 if (0 == c_strcasecmp(hdrval, "x-compress"))
3769 hs->local_encoding = ENC_COMPRESS;
3770 else if (0 == c_strcasecmp(hdrval, "x-gzip"))
3771 hs->local_encoding = ENC_GZIP;
3772 break;
3773 case '\0':
3774 hs->local_encoding = ENC_NONE;
3775 }
3776
3777 if (hs->local_encoding == ENC_INVALID)
3778 {
3779 DEBUGP (("Unrecognized Content-Encoding: %s\n", hdrval));
3780 hs->local_encoding = ENC_NONE;
3781 }
3782 #ifdef HAVE_LIBZ
3783 else if (hs->local_encoding == ENC_GZIP
3784 && opt.compression != compression_none)
3785 {
3786 const char *p;
3787
3788 /* Make sure the Content-Type is not gzip before decompressing */
3789 if (type)
3790 {
3791 p = strchr (type, '/');
3792 if (p == NULL)
3793 {
3794 hs->remote_encoding = ENC_GZIP;
3795 hs->local_encoding = ENC_NONE;
3796 }
3797 else
3798 {
3799 p++;
3800 if (c_tolower(p[0]) == 'x' && p[1] == '-')
3801 p += 2;
3802 if (0 != c_strcasecmp (p, "gzip"))
3803 {
3804 hs->remote_encoding = ENC_GZIP;
3805 hs->local_encoding = ENC_NONE;
3806 }
3807 }
3808 }
3809 else
3810 {
3811 hs->remote_encoding = ENC_GZIP;
3812 hs->local_encoding = ENC_NONE;
3813 }
3814
3815 /* don't uncompress if a file ends with '.gz' or '.tgz' */
3816 if (hs->remote_encoding == ENC_GZIP
3817 && (p = strrchr(u->file, '.'))
3818 && (c_strcasecmp(p, ".gz") == 0 || c_strcasecmp(p, ".tgz") == 0))
3819 {
3820 DEBUGP (("Enabling broken server workaround. Will not decompress this GZip file.\n"));
3821 hs->remote_encoding = ENC_NONE;
3822 }
3823 }
3824 #endif
3825 }
3826
3827 /* 20x responses are counted among successful by default. */
3828 if (H_20X (statcode))
3829 *dt |= RETROKF;
3830
3831 if (statcode == HTTP_STATUS_NO_CONTENT)
3832 {
3833 /* 204 response has no body (RFC 2616, 4.3) */
3834
3835 /* In case the caller cares to look... */
3836 hs->len = 0;
3837 hs->res = 0;
3838 hs->restval = 0;
3839
3840 CLOSE_FINISH (sock);
3841
3842 retval = RETRFINISHED;
3843 goto cleanup;
3844 }
3845
3846 /* Return if redirected. */
3847 if (H_REDIRECTED (statcode) || statcode == HTTP_STATUS_MULTIPLE_CHOICES)
3848 {
3849 /* RFC2068 says that in case of the 300 (multiple choices)
3850 response, the server can output a preferred URL through
3851 `Location' header; otherwise, the request should be treated
3852 like GET. So, if the location is set, it will be a
3853 redirection; otherwise, just proceed normally. */
3854 if (statcode == HTTP_STATUS_MULTIPLE_CHOICES && !hs->newloc)
3855 *dt |= RETROKF;
3856 else
3857 {
3858 logprintf (LOG_VERBOSE,
3859 _("Location: %s%s\n"),
3860 hs->newloc ? escnonprint_uri (hs->newloc) : _("unspecified"),
3861 hs->newloc ? _(" [following]") : "");
3862
3863 /* In case the caller cares to look... */
3864 hs->len = 0;
3865 hs->res = 0;
3866 hs->restval = 0;
3867
3868 /* Normally we are not interested in the response body of a redirect.
3869 But if we are writing a WARC file we are: we like to keep everything. */
3870 if (warc_enabled)
3871 {
3872 int _err = read_response_body (hs, sock, NULL, contlen, 0,
3873 chunked_transfer_encoding,
3874 u->url, warc_timestamp_str,
3875 warc_request_uuid, warc_ip, type,
3876 statcode, head);
3877
3878 if (_err != RETRFINISHED || hs->res < 0)
3879 {
3880 CLOSE_INVALIDATE (sock);
3881 retval = _err;
3882 goto cleanup;
3883 }
3884 else
3885 CLOSE_FINISH (sock);
3886 }
3887 else
3888 {
3889 /* Since WARC is disabled, we are not interested in the response body. */
3890 if (keep_alive && !head_only
3891 && skip_short_body (sock, contlen, chunked_transfer_encoding))
3892 CLOSE_FINISH (sock);
3893 else
3894 CLOSE_INVALIDATE (sock);
3895 }
3896
3897 /* From RFC2616: The status codes 303 and 307 have
3898 been added for servers that wish to make unambiguously
3899 clear which kind of reaction is expected of the client.
3900
3901 A 307 should be redirected using the same method,
3902 in other words, a POST should be preserved and not
3903 converted to a GET in that case.
3904
3905 With strict adherence to RFC2616, POST requests are not
3906 converted to a GET request on 301 Permanent Redirect
3907 or 302 Temporary Redirect.
3908
3909 A switch may be provided later based on the HTTPbis draft
3910 that allows clients to convert POST requests to GET
3911 requests on 301 and 302 response codes. */
3912 switch (statcode)
3913 {
3914 case HTTP_STATUS_TEMPORARY_REDIRECT:
3915 case HTTP_STATUS_PERMANENT_REDIRECT:
3916 retval = NEWLOCATION_KEEP_POST;
3917 goto cleanup;
3918 case HTTP_STATUS_MOVED_PERMANENTLY:
3919 if (opt.method && c_strcasecmp (opt.method, "post") != 0)
3920 {
3921 retval = NEWLOCATION_KEEP_POST;
3922 goto cleanup;
3923 }
3924 break;
3925 case HTTP_STATUS_MOVED_TEMPORARILY:
3926 if (opt.method && c_strcasecmp (opt.method, "post") != 0)
3927 {
3928 retval = NEWLOCATION_KEEP_POST;
3929 goto cleanup;
3930 }
3931 break;
3932 }
3933 retval = NEWLOCATION;
3934 goto cleanup;
3935 }
3936 }
3937
3938 if (cond_get)
3939 {
3940 if (statcode == HTTP_STATUS_NOT_MODIFIED)
3941 {
3942 logprintf (LOG_VERBOSE,
3943 _ ("File %s not modified on server. Omitting download.\n\n"),
3944 quote (hs->local_file));
3945 *dt |= RETROKF;
3946 CLOSE_FINISH (sock);
3947 retval = RETRUNNEEDED;
3948 goto cleanup;
3949 }
3950 }
3951
3952 set_content_type (dt, type);
3953
3954 if (opt.adjust_extension)
3955 {
3956 const char *encoding_ext = NULL;
3957 switch (hs->local_encoding)
3958 {
3959 case ENC_INVALID:
3960 case ENC_NONE:
3961 break;
3962 case ENC_BROTLI:
3963 encoding_ext = ".br";
3964 break;
3965 case ENC_COMPRESS:
3966 encoding_ext = ".Z";
3967 break;
3968 case ENC_DEFLATE:
3969 encoding_ext = ".zlib";
3970 break;
3971 case ENC_GZIP:
3972 encoding_ext = ".gz";
3973 break;
3974 default:
3975 DEBUGP (("No extension found for encoding %d\n",
3976 hs->local_encoding));
3977 }
3978 if (encoding_ext != NULL)
3979 {
3980 char *file_ext = strrchr (hs->local_file, '.');
3981 /* strip Content-Encoding extension (it will be re-added later) */
3982 if (file_ext != NULL && 0 == strcasecmp (file_ext, encoding_ext))
3983 *file_ext = '\0';
3984 }
3985 if (*dt & TEXTHTML)
3986 /* -E / --adjust-extension / adjust_extension = on was specified,
3987 and this is a text/html file. If some case-insensitive
3988 variation on ".htm[l]" isn't already the file's suffix,
3989 tack on ".html". */
3990 {
3991 ensure_extension (hs, ".html", dt);
3992 }
3993 else if (*dt & TEXTCSS)
3994 {
3995 ensure_extension (hs, ".css", dt);
3996 }
3997 if (encoding_ext != NULL)
3998 {
3999 ensure_extension (hs, encoding_ext, dt);
4000 }
4001 }
4002
4003 if (cond_get)
4004 {
4005 /* Handle the case when server ignores If-Modified-Since header. */
4006 if (statcode == HTTP_STATUS_OK && hs->remote_time)
4007 {
4008 time_t tmr = http_atotm (hs->remote_time);
4009
4010 /* Check if the local file is up-to-date based on Last-Modified header
4011 and content length. */
4012 if (tmr != (time_t) - 1 && tmr <= hs->orig_file_tstamp
4013 && (contlen == -1 || contlen == hs->orig_file_size))
4014 {
4015 logprintf (LOG_VERBOSE,
4016 _("Server ignored If-Modified-Since header for file %s.\n"
4017 "You might want to add --no-if-modified-since option."
4018 "\n\n"),
4019 quote (hs->local_file));
4020 *dt |= RETROKF;
4021 CLOSE_INVALIDATE (sock);
4022 retval = RETRUNNEEDED;
4023 goto cleanup;
4024 }
4025 }
4026 }
4027
4028 if (statcode == HTTP_STATUS_RANGE_NOT_SATISFIABLE
4029 || (!opt.timestamping && hs->restval > 0 && statcode == HTTP_STATUS_OK
4030 && contrange == 0 && contlen >= 0 && hs->restval >= contlen))
4031 {
4032 /* If `-c' is in use and the file has been fully downloaded (or
4033 the remote file has shrunk), Wget effectively requests bytes
4034 after the end of file and the server response with 416
4035 (or 200 with a <= Content-Length. */
4036 logputs (LOG_VERBOSE, _("\
4037 \n The file is already fully retrieved; nothing to do.\n\n"));
4038 /* In case the caller inspects. */
4039 hs->len = contlen;
4040 hs->res = 0;
4041 /* Mark as successfully retrieved. */
4042 *dt |= RETROKF;
4043
4044 /* Try to maintain the keep-alive connection. It is often cheaper to
4045 * consume some bytes which have already been sent than to negotiate
4046 * a new connection. However, if the body is too large, or we don't
4047 * care about keep-alive, then simply terminate the connection */
4048 if (keep_alive &&
4049 skip_short_body (sock, contlen, chunked_transfer_encoding))
4050 CLOSE_FINISH (sock);
4051 else
4052 CLOSE_INVALIDATE (sock);
4053 retval = RETRUNNEEDED;
4054 goto cleanup;
4055 }
4056 if ((contrange != 0 && contrange != hs->restval)
4057 || (H_PARTIAL (statcode) && !contrange && hs->restval))
4058 {
4059 /* The Range request was somehow misunderstood by the server.
4060 Bail out. */
4061 CLOSE_INVALIDATE (sock);
4062 retval = RANGEERR;
4063 goto cleanup;
4064 }
4065 if (contlen == -1)
4066 hs->contlen = -1;
4067 /* If the response is gzipped, the uncompressed size is unknown. */
4068 else if (hs->remote_encoding == ENC_GZIP)
4069 hs->contlen = -1;
4070 else
4071 hs->contlen = contlen + contrange;
4072
4073 if (opt.verbose)
4074 {
4075 if (*dt & RETROKF)
4076 {
4077 /* No need to print this output if the body won't be
4078 downloaded at all, or if the original server response is
4079 printed. */
4080 logputs (LOG_VERBOSE, _("Length: "));
4081 if (contlen != -1)
4082 {
4083 logputs (LOG_VERBOSE, number_to_static_string (contlen + contrange));
4084 if (contlen + contrange >= 1024)
4085 logprintf (LOG_VERBOSE, " (%s)",
4086 human_readable (contlen + contrange, 10, 1));
4087 if (contrange)
4088 {
4089 if (contlen >= 1024)
4090 logprintf (LOG_VERBOSE, _(", %s (%s) remaining"),
4091 number_to_static_string (contlen),
4092 human_readable (contlen, 10, 1));
4093 else
4094 logprintf (LOG_VERBOSE, _(", %s remaining"),
4095 number_to_static_string (contlen));
4096 }
4097 }
4098 else
4099 logputs (LOG_VERBOSE,
4100 opt.ignore_length ? _("ignored") : _("unspecified"));
4101 if (type)
4102 logprintf (LOG_VERBOSE, " [%s]\n", quotearg_style (escape_quoting_style, type));
4103 else
4104 logputs (LOG_VERBOSE, "\n");
4105 }
4106 }
4107
4108 /* Return if we have no intention of further downloading. */
4109 if ((!(*dt & RETROKF) && !opt.content_on_error) || head_only || (opt.spider && !opt.recursive))
4110 {
4111 /* In case the caller cares to look... */
4112 hs->len = 0;
4113 hs->res = 0;
4114 hs->restval = 0;
4115
4116 /* Normally we are not interested in the response body of a error responses.
4117 But if we are writing a WARC file we are: we like to keep everything. */
4118 if (warc_enabled)
4119 {
4120 int _err = read_response_body (hs, sock, NULL, contlen, 0,
4121 chunked_transfer_encoding,
4122 u->url, warc_timestamp_str,
4123 warc_request_uuid, warc_ip, type,
4124 statcode, head);
4125
4126 if (_err != RETRFINISHED || hs->res < 0)
4127 {
4128 CLOSE_INVALIDATE (sock);
4129 retval = _err;
4130 goto cleanup;
4131 }
4132
4133 CLOSE_FINISH (sock);
4134 }
4135 else
4136 {
4137 /* Since WARC is disabled, we are not interested in the response body. */
4138 if (head_only)
4139 /* Pre-1.10 Wget used CLOSE_INVALIDATE here. Now we trust the
4140 servers not to send body in response to a HEAD request, and
4141 those that do will likely be caught by test_socket_open.
4142 If not, they can be worked around using
4143 `--no-http-keep-alive'. */
4144 CLOSE_FINISH (sock);
4145 else if (opt.spider && !opt.recursive)
4146 /* we just want to see if the page exists - no downloading required */
4147 CLOSE_INVALIDATE (sock);
4148 else if (keep_alive
4149 && skip_short_body (sock, contlen, chunked_transfer_encoding))
4150 /* Successfully skipped the body; also keep using the socket. */
4151 CLOSE_FINISH (sock);
4152 else
4153 CLOSE_INVALIDATE (sock);
4154 }
4155
4156 if (statcode == HTTP_STATUS_GATEWAY_TIMEOUT)
4157 retval = GATEWAYTIMEOUT;
4158 else
4159 retval = RETRFINISHED;
4160
4161 goto cleanup;
4162 }
4163
4164 err = open_output_stream (hs, count, &fp);
4165 if (err != RETROK)
4166 {
4167 /* Make sure that errno doesn't get clobbered.
4168 * This is the case for OpenSSL's SSL_shutdown(). */
4169 int tmp_errno = errno;
4170 CLOSE_INVALIDATE (sock);
4171 errno = tmp_errno;
4172 retval = err;
4173 goto cleanup;
4174 }
4175
4176 #ifdef ENABLE_XATTR
4177 if (opt.enable_xattr)
4178 {
4179 if (original_url != u)
4180 set_file_metadata (u, original_url, fp);
4181 else
4182 set_file_metadata (u, NULL, fp);
4183 }
4184 #endif
4185
4186 err = read_response_body (hs, sock, fp, contlen, contrange,
4187 chunked_transfer_encoding,
4188 u->url, warc_timestamp_str,
4189 warc_request_uuid, warc_ip, type,
4190 statcode, head);
4191
4192 if (hs->res >= 0)
4193 CLOSE_FINISH (sock);
4194 else
4195 CLOSE_INVALIDATE (sock);
4196
4197 if (!output_stream)
4198 fclose (fp);
4199
4200 retval = err;
4201
4202 cleanup:
4203 xfree (head);
4204 xfree (type);
4205 xfree (message);
4206 resp_free (&resp);
4207 request_free (&req);
4208
4209 return retval;
4210 }
4211
4212 /* Check whether the supplied HTTP status code is among those
4213 listed for the --retry-on-http-error option. */
4214 static bool
4215 check_retry_on_http_error (const int statcode)
4216 {
4217 const char *tok = opt.retry_on_http_error;
4218 while (tok && *tok)
4219 {
4220 if (atoi (tok) == statcode)
4221 return true;
4222 if ((tok = strchr (tok, ',')))
4223 ++tok;
4224 }
4225 return false;
4226 }
4227
4228 /* The genuine HTTP loop! This is the part where the retrieval is
4229 retried, and retried, and retried, and... */
4230 uerr_t
4231 http_loop (const struct url *u, struct url *original_url, char **newloc,
4232 char **local_file, const char *referer, int *dt, struct url *proxy,
4233 struct iri *iri)
4234 {
4235 int count;
4236 bool got_head = false; /* used for time-stamping and filename detection */
4237 bool time_came_from_head = false;
4238 bool got_name = false;
4239 char *tms;
4240 const char *tmrate;
4241 uerr_t err, ret = TRYLIMEXC;
4242 time_t tmr = -1; /* remote time-stamp */
4243 struct http_stat hstat; /* HTTP status */
4244 struct stat st;
4245 bool send_head_first = true;
4246 bool force_full_retrieve = false;
4247
4248 /* If we are writing to a WARC file: always retrieve the whole file. */
4249 if (opt.warc_filename != NULL)
4250 force_full_retrieve = true;
4251
4252 /* Assert that no value for *LOCAL_FILE was passed. */
4253 assert (local_file == NULL || *local_file == NULL);
4254
4255 /* Reset NEWLOC parameter. */
4256 *newloc = NULL;
4257
4258 /* This used to be done in main, but it's a better idea to do it
4259 here so that we don't go through the hoops if we're just using
4260 FTP or whatever. */
4261 if (opt.cookies)
4262 load_cookies ();
4263
4264 /* Warn on (likely bogus) wildcard usage in HTTP. */
4265 if (opt.ftp_glob && has_wildcards_p (u->path))
4266 logputs (LOG_VERBOSE, _("Warning: wildcards not supported in HTTP.\n"));
4267
4268 /* Setup hstat struct. */
4269 xzero (hstat);
4270 hstat.referer = referer;
4271
4272 if (opt.output_document)
4273 {
4274 hstat.local_file = xstrdup (opt.output_document);
4275 got_name = true;
4276 }
4277 else if (!opt.content_disposition)
4278 {
4279 hstat.local_file =
4280 url_file_name (opt.trustservernames ? u : original_url, NULL);
4281 got_name = true;
4282 }
4283
4284 if (got_name && file_exists_p (hstat.local_file, NULL) && opt.noclobber && !opt.output_document)
4285 {
4286 /* If opt.noclobber is turned on and file already exists, do not
4287 retrieve the file. But if the output_document was given, then this
4288 test was already done and the file didn't exist. Hence the !opt.output_document */
4289 get_file_flags (hstat.local_file, dt);
4290 ret = RETROK;
4291 goto exit;
4292 }
4293
4294 /* Reset the counter. */
4295 count = 0;
4296
4297 /* Reset the document type. */
4298 *dt = 0;
4299
4300 /* Skip preliminary HEAD request if we're not in spider mode. */
4301 if (!opt.spider)
4302 send_head_first = false;
4303
4304 /* Send preliminary HEAD request if --content-disposition and -c are used
4305 together. */
4306 if (opt.content_disposition && opt.always_rest)
4307 send_head_first = true;
4308
4309 #ifdef HAVE_METALINK
4310 if (opt.metalink_over_http)
4311 {
4312 *dt |= METALINK_METADATA;
4313 send_head_first = true;
4314 }
4315 #endif
4316
4317 if (opt.timestamping)
4318 {
4319 /* Use conditional get request if requested
4320 * and if timestamp is known at this moment. */
4321 if (opt.if_modified_since && !send_head_first && got_name && file_exists_p (hstat.local_file, NULL))
4322 {
4323 *dt |= IF_MODIFIED_SINCE;
4324 uerr_t timestamp_err = set_file_timestamp (&hstat);
4325 if (timestamp_err != RETROK)
4326 {
4327 ret = timestamp_err;
4328 goto exit;
4329 }
4330 }
4331 /* Send preliminary HEAD request if -N is given and we have existing
4332 * destination file or content disposition is enabled. */
4333 else if (opt.content_disposition || file_exists_p (hstat.local_file, NULL))
4334 send_head_first = true;
4335 }
4336
4337 /* THE loop */
4338 do
4339 {
4340 /* Increment the pass counter. */
4341 ++count;
4342 sleep_between_retrievals (count);
4343
4344 /* Get the current time string. */
4345 tms = datetime_str (time (NULL));
4346
4347 if (opt.spider && !got_head)
4348 logprintf (LOG_VERBOSE,
4349 _("Spider mode enabled. Check if remote file exists.\n"));
4350
4351 /* Print fetch message, if opt.verbose. */
4352 if (opt.verbose)
4353 {
4354 char *hurl = url_string (u, URL_AUTH_HIDE_PASSWD);
4355
4356 if (count > 1)
4357 {
4358 char tmp[256];
4359 sprintf (tmp, _("(try:%2d)"), count);
4360 logprintf (LOG_NOTQUIET, "--%s-- %s %s\n",
4361 tms, tmp, hurl);
4362 }
4363 else
4364 {
4365 logprintf (LOG_NOTQUIET, "--%s-- %s\n",
4366 tms, hurl);
4367 }
4368
4369 #ifdef WINDOWS
4370 ws_changetitle (hurl);
4371 #endif
4372 xfree (hurl);
4373 }
4374
4375 /* Default document type is empty. However, if spider mode is
4376 on or time-stamping is employed, HEAD_ONLY commands is
4377 encoded within *dt. */
4378 if (send_head_first && !got_head)
4379 *dt |= HEAD_ONLY;
4380 else
4381 *dt &= ~HEAD_ONLY;
4382
4383 /* Decide whether or not to restart. */
4384 if (force_full_retrieve)
4385 hstat.restval = hstat.len;
4386 else if (opt.start_pos >= 0)
4387 hstat.restval = opt.start_pos;
4388 else if (opt.always_rest
4389 && got_name
4390 && stat (hstat.local_file, &st) == 0
4391 && S_ISREG (st.st_mode))
4392 /* When -c is used, continue from on-disk size. (Can't use
4393 hstat.len even if count>1 because we don't want a failed
4394 first attempt to clobber existing data.) */
4395 hstat.restval = st.st_size;
4396 else if (count > 1)
4397 {
4398 /* otherwise, continue where the previous try left off */
4399 if (hstat.len < hstat.restval)
4400 hstat.restval -= hstat.len;
4401 else
4402 hstat.restval = hstat.len;
4403 }
4404 else
4405 hstat.restval = 0;
4406
4407 /* Decide whether to send the no-cache directive. We send it in
4408 two cases:
4409 a) we're using a proxy, and we're past our first retrieval.
4410 Some proxies are notorious for caching incomplete data, so
4411 we require a fresh get.
4412 b) caching is explicitly inhibited. */
4413 if ((proxy && count > 1) /* a */
4414 || !opt.allow_cache) /* b */
4415 *dt |= SEND_NOCACHE;
4416 else
4417 *dt &= ~SEND_NOCACHE;
4418
4419 /* Try fetching the document, or at least its head. */
4420 err = gethttp (u, original_url, &hstat, dt, proxy, iri, count);
4421
4422 /* Time? */
4423 tms = datetime_str (time (NULL));
4424
4425 /* Get the new location (with or without the redirection). */
4426 if (hstat.newloc)
4427 *newloc = xstrdup (hstat.newloc);
4428
4429 switch (err)
4430 {
4431 case HERR: case HEOF: case CONSOCKERR:
4432 case CONERROR: case READERR: case WRITEFAILED:
4433 case RANGEERR: case FOPEN_EXCL_ERR: case GATEWAYTIMEOUT:
4434 /* Non-fatal errors continue executing the loop, which will
4435 bring them to "while" statement at the end, to judge
4436 whether the number of tries was exceeded. */
4437 printwhat (count, opt.ntry);
4438 continue;
4439 case FWRITEERR: case FOPENERR:
4440 /* Another fatal error. */
4441 logputs (LOG_VERBOSE, "\n");
4442 logprintf (LOG_NOTQUIET, _("Cannot write to %s (%s).\n"),
4443 quote (hstat.local_file), strerror (errno));
4444 ret = err;
4445 goto exit;
4446 case HOSTERR:
4447 /* Fatal unless option set otherwise. */
4448 if ( opt.retry_on_host_error )
4449 {
4450 printwhat (count, opt.ntry);
4451 continue;
4452 }
4453 ret = err;
4454 goto exit;
4455 case CONIMPOSSIBLE: case PROXERR: case SSLINITFAILED:
4456 case CONTNOTSUPPORTED: case VERIFCERTERR: case FILEBADFILE:
4457 case UNKNOWNATTR:
4458 /* Fatal errors just return from the function. */
4459 ret = err;
4460 goto exit;
4461 case ATTRMISSING:
4462 /* A missing attribute in a Header is a fatal Protocol error. */
4463 logputs (LOG_VERBOSE, "\n");
4464 logprintf (LOG_NOTQUIET, _("Required attribute missing from Header received.\n"));
4465 ret = err;
4466 goto exit;
4467 case AUTHFAILED:
4468 logputs (LOG_VERBOSE, "\n");
4469 logprintf (LOG_NOTQUIET, _("Username/Password Authentication Failed.\n"));
4470 ret = err;
4471 goto exit;
4472 case WARC_ERR:
4473 /* A fatal WARC error. */
4474 logputs (LOG_VERBOSE, "\n");
4475 logprintf (LOG_NOTQUIET, _("Cannot write to WARC file.\n"));
4476 ret = err;
4477 goto exit;
4478 case WARC_TMP_FOPENERR: case WARC_TMP_FWRITEERR:
4479 /* A fatal WARC error. */
4480 logputs (LOG_VERBOSE, "\n");
4481 logprintf (LOG_NOTQUIET, _("Cannot write to temporary WARC file.\n"));
4482 ret = err;
4483 goto exit;
4484 case CONSSLERR:
4485 /* Another fatal error. */
4486 logprintf (LOG_NOTQUIET, _("Unable to establish SSL connection.\n"));
4487 ret = err;
4488 goto exit;
4489 case UNLINKERR:
4490 /* Another fatal error. */
4491 logputs (LOG_VERBOSE, "\n");
4492 logprintf (LOG_NOTQUIET, _("Cannot unlink %s (%s).\n"),
4493 quote (hstat.local_file), strerror (errno));
4494 ret = err;
4495 goto exit;
4496 case NEWLOCATION:
4497 case NEWLOCATION_KEEP_POST:
4498 /* Return the new location to the caller. */
4499 if (!*newloc)
4500 {
4501 logprintf (LOG_NOTQUIET,
4502 _("ERROR: Redirection (%d) without location.\n"),
4503 hstat.statcode);
4504 ret = WRONGCODE;
4505 }
4506 else
4507 {
4508 ret = err;
4509 }
4510 goto exit;
4511 case RETRUNNEEDED:
4512 /* The file was already fully retrieved. */
4513 ret = RETROK;
4514 goto exit;
4515 case RETRFINISHED:
4516 /* Deal with you later. */
4517 break;
4518 #ifdef HAVE_METALINK
4519 case RETR_WITH_METALINK:
4520 {
4521 if (hstat.metalink == NULL)
4522 {
4523 logputs (LOG_NOTQUIET,
4524 _("Could not find Metalink data in HTTP response. "
4525 "Downloading file using HTTP GET.\n"));
4526 *dt &= ~METALINK_METADATA;
4527 *dt &= ~HEAD_ONLY;
4528 got_head = true;
4529 continue;
4530 }
4531
4532 logputs (LOG_VERBOSE,
4533 _("Metalink headers found. "
4534 "Switching to Metalink mode.\n"));
4535
4536 ret = retrieve_from_metalink (hstat.metalink);
4537 goto exit;
4538 }
4539 break;
4540 #endif
4541 default:
4542 /* All possibilities should have been exhausted. */
4543 abort ();
4544 }
4545
4546 if (!(*dt & RETROKF))
4547 {
4548 char *hurl = NULL;
4549 if (!opt.verbose)
4550 {
4551 /* #### Ugly ugly ugly! */
4552 hurl = url_string (u, URL_AUTH_HIDE_PASSWD);
4553 logprintf (LOG_NONVERBOSE, "%s:\n", hurl);
4554 }
4555
4556 /* Fall back to GET if HEAD fails with a 500 or 501 error code. */
4557 if (*dt & HEAD_ONLY
4558 && (hstat.statcode == 500 || hstat.statcode == 501))
4559 {
4560 got_head = true;
4561 xfree (hurl);
4562 continue;
4563 }
4564 /* Maybe we should always keep track of broken links, not just in
4565 * spider mode.
4566 * Don't log error if it was UTF-8 encoded because we will try
4567 * once unencoded. */
4568 else if (opt.spider && !iri->utf8_encode)
4569 {
4570 /* #### Again: ugly ugly ugly! */
4571 if (!hurl)
4572 hurl = url_string (u, URL_AUTH_HIDE_PASSWD);
4573 nonexisting_url (hurl);
4574 logprintf (LOG_NOTQUIET, _("\
4575 Remote file does not exist -- broken link!!!\n"));
4576 }
4577 else if (check_retry_on_http_error (hstat.statcode))
4578 {
4579 printwhat (count, opt.ntry);
4580 xfree (hurl);
4581 continue;
4582 }
4583 else
4584 {
4585 logprintf (LOG_NOTQUIET, _("%s ERROR %d: %s.\n"),
4586 tms, hstat.statcode,
4587 quotearg_style (escape_quoting_style, hstat.error));
4588 }
4589 logputs (LOG_VERBOSE, "\n");
4590 ret = WRONGCODE;
4591 xfree (hurl);
4592 goto exit;
4593 }
4594
4595 /* Did we get the time-stamp? */
4596 if (!got_head || (opt.spider && !opt.recursive))
4597 {
4598 got_head = true; /* no more time-stamping */
4599
4600 if (opt.timestamping && !hstat.remote_time)
4601 {
4602 logputs (LOG_NOTQUIET, _("\
4603 Last-modified header missing -- time-stamps turned off.\n"));
4604 }
4605 else if (hstat.remote_time)
4606 {
4607 /* Convert the date-string into struct tm. */
4608 tmr = http_atotm (hstat.remote_time);
4609 if (tmr == (time_t) (-1))
4610 logputs (LOG_VERBOSE, _("\
4611 Last-modified header invalid -- time-stamp ignored.\n"));
4612 if (*dt & HEAD_ONLY)
4613 time_came_from_head = true;
4614 }
4615
4616 if (send_head_first)
4617 {
4618 /* The time-stamping section. */
4619 if (opt.timestamping)
4620 {
4621 if (hstat.orig_file_name) /* Perform the following
4622 checks only if the file
4623 we're supposed to
4624 download already exists. */
4625 {
4626 if (hstat.remote_time &&
4627 tmr != (time_t) (-1))
4628 {
4629 /* Now time-stamping can be used validly.
4630 Time-stamping means that if the sizes of
4631 the local and remote file match, and local
4632 file is newer than the remote file, it will
4633 not be retrieved. Otherwise, the normal
4634 download procedure is resumed. */
4635 if (hstat.orig_file_tstamp >= tmr)
4636 {
4637 if (hstat.contlen == -1
4638 || hstat.orig_file_size == hstat.contlen)
4639 {
4640 logprintf (LOG_VERBOSE, _("\
4641 Server file no newer than local file %s -- not retrieving.\n\n"),
4642 quote (hstat.orig_file_name));
4643 ret = RETROK;
4644 goto exit;
4645 }
4646 else
4647 {
4648 logprintf (LOG_VERBOSE, _("\
4649 The sizes do not match (local %s) -- retrieving.\n"),
4650 number_to_static_string (hstat.orig_file_size));
4651 }
4652 }
4653 else
4654 {
4655 force_full_retrieve = true;
4656 logputs (LOG_VERBOSE,
4657 _("Remote file is newer, retrieving.\n"));
4658 }
4659
4660 logputs (LOG_VERBOSE, "\n");
4661 }
4662 }
4663
4664 /* free_hstat (&hstat); */
4665 hstat.timestamp_checked = true;
4666 }
4667
4668 if (opt.spider)
4669 {
4670 bool finished = true;
4671 if (opt.recursive)
4672 {
4673 if ((*dt & TEXTHTML) || (*dt & TEXTCSS))
4674 {
4675 logputs (LOG_VERBOSE, _("\
4676 Remote file exists and could contain links to other resources -- retrieving.\n\n"));
4677 finished = false;
4678 }
4679 else
4680 {
4681 logprintf (LOG_VERBOSE, _("\
4682 Remote file exists but does not contain any link -- not retrieving.\n\n"));
4683 ret = RETROK; /* RETRUNNEEDED is not for caller. */
4684 }
4685 }
4686 else
4687 {
4688 if ((*dt & TEXTHTML) || (*dt & TEXTCSS))
4689 {
4690 logprintf (LOG_VERBOSE, _("\
4691 Remote file exists and could contain further links,\n\
4692 but recursion is disabled -- not retrieving.\n\n"));
4693 }
4694 else
4695 {
4696 logprintf (LOG_VERBOSE, _("\
4697 Remote file exists.\n\n"));
4698 }
4699 ret = RETROK; /* RETRUNNEEDED is not for caller. */
4700 }
4701
4702 if (finished)
4703 {
4704 logprintf (LOG_NONVERBOSE,
4705 _("%s URL: %s %2d %s\n"),
4706 tms, u->url, hstat.statcode,
4707 hstat.message ? quotearg_style (escape_quoting_style, hstat.message) : "");
4708 goto exit;
4709 }
4710 }
4711
4712 got_name = true;
4713 *dt &= ~HEAD_ONLY;
4714 count = 0; /* the retrieve count for HEAD is reset */
4715 continue;
4716 } /* send_head_first */
4717 } /* !got_head */
4718
4719 if (opt.useservertimestamps
4720 && (tmr != (time_t) (-1))
4721 && ((hstat.len == hstat.contlen) ||
4722 ((hstat.res == 0) && (hstat.contlen == -1))))
4723 {
4724 const char *fl = NULL;
4725 set_local_file (&fl, hstat.local_file);
4726 if (fl)
4727 {
4728 time_t newtmr = -1;
4729 /* Reparse time header, in case it's changed. */
4730 if (time_came_from_head
4731 && hstat.remote_time && hstat.remote_time[0])
4732 {
4733 newtmr = http_atotm (hstat.remote_time);
4734 if (newtmr != (time_t)-1)
4735 tmr = newtmr;
4736 }
4737 touch (fl, tmr);
4738 }
4739 }
4740 /* End of time-stamping section. */
4741
4742 tmrate = retr_rate (hstat.rd_size, hstat.dltime);
4743 total_download_time += hstat.dltime;
4744
4745 if (hstat.len == hstat.contlen)
4746 {
4747 if (*dt & RETROKF || opt.content_on_error)
4748 {
4749 bool write_to_stdout = (opt.output_document && HYPHENP (opt.output_document));
4750
4751 logprintf (LOG_VERBOSE,
4752 write_to_stdout
4753 ? _("%s (%s) - written to stdout %s[%s/%s]\n\n")
4754 : _("%s (%s) - %s saved [%s/%s]\n\n"),
4755 tms, tmrate,
4756 write_to_stdout ? "" : quote (hstat.local_file),
4757 number_to_static_string (hstat.len),
4758 number_to_static_string (hstat.contlen));
4759 logprintf (LOG_NONVERBOSE,
4760 "%s URL:%s [%s/%s] -> \"%s\" [%d]\n",
4761 tms, u->url,
4762 number_to_static_string (hstat.len),
4763 number_to_static_string (hstat.contlen),
4764 hstat.local_file, count);
4765 }
4766 ++numurls;
4767 total_downloaded_bytes += hstat.rd_size;
4768
4769 /* Remember that we downloaded the file for later ".orig" code. */
4770 if (*dt & ADDED_HTML_EXTENSION)
4771 downloaded_file (FILE_DOWNLOADED_AND_HTML_EXTENSION_ADDED, hstat.local_file);
4772 else
4773 downloaded_file (FILE_DOWNLOADED_NORMALLY, hstat.local_file);
4774
4775 ret = RETROK;
4776 goto exit;
4777 }
4778 else if (hstat.res == 0) /* No read error */
4779 {
4780 if (hstat.contlen == -1) /* We don't know how much we were supposed
4781 to get, so assume we succeeded. */
4782 {
4783 if (*dt & RETROKF || opt.content_on_error)
4784 {
4785 bool write_to_stdout = (opt.output_document && HYPHENP (opt.output_document));
4786
4787 logprintf (LOG_VERBOSE,
4788 write_to_stdout
4789 ? _("%s (%s) - written to stdout %s[%s]\n\n")
4790 : _("%s (%s) - %s saved [%s]\n\n"),
4791 tms, tmrate,
4792 write_to_stdout ? "" : quote (hstat.local_file),
4793 number_to_static_string (hstat.len));
4794 if (!(opt.verbose || opt.quiet))
4795 {
4796 char *url = url_string (u, URL_AUTH_HIDE_PASSWD);
4797 logprintf (LOG_NONVERBOSE,
4798 "%s URL:%s [%s] -> \"%s\" [%d]\n",
4799 tms, url, number_to_static_string (hstat.len),
4800 hstat.local_file, count);
4801 xfree (url);
4802 }
4803 }
4804 ++numurls;
4805 total_downloaded_bytes += hstat.rd_size;
4806
4807 /* Remember that we downloaded the file for later ".orig" code. */
4808 if (*dt & ADDED_HTML_EXTENSION)
4809 downloaded_file (FILE_DOWNLOADED_AND_HTML_EXTENSION_ADDED, hstat.local_file);
4810 else
4811 downloaded_file (FILE_DOWNLOADED_NORMALLY, hstat.local_file);
4812
4813 ret = RETROK;
4814 goto exit;
4815 }
4816 else if (hstat.len < hstat.contlen) /* meaning we lost the
4817 connection too soon */
4818 {
4819 logprintf (LOG_VERBOSE,
4820 _("%s (%s) - Connection closed at byte %s. "),
4821 tms, tmrate, number_to_static_string (hstat.len));
4822 printwhat (count, opt.ntry);
4823 continue;
4824 }
4825 else if (hstat.len != hstat.restval)
4826 /* Getting here would mean reading more data than
4827 requested with content-length, which we never do. */
4828 abort ();
4829 else
4830 {
4831 /* Getting here probably means that the content-length was
4832 * _less_ than the original, local size. We should probably
4833 * truncate or re-read, or something. FIXME */
4834 ret = RETROK;
4835 goto exit;
4836 }
4837 }
4838 else /* from now on hstat.res can only be -1 */
4839 {
4840 if (hstat.contlen == -1)
4841 {
4842 logprintf (LOG_VERBOSE,
4843 _("%s (%s) - Read error at byte %s (%s)."),
4844 tms, tmrate, number_to_static_string (hstat.len),
4845 hstat.rderrmsg);
4846 printwhat (count, opt.ntry);
4847 continue;
4848 }
4849 else /* hstat.res == -1 and contlen is given */
4850 {
4851 logprintf (LOG_VERBOSE,
4852 _("%s (%s) - Read error at byte %s/%s (%s). "),
4853 tms, tmrate,
4854 number_to_static_string (hstat.len),
4855 number_to_static_string (hstat.contlen),
4856 hstat.rderrmsg);
4857 printwhat (count, opt.ntry);
4858 continue;
4859 }
4860 }
4861 /* not reached */
4862 }
4863 while (!opt.ntry || (count < opt.ntry));
4864
4865 exit:
4866 if ((ret == RETROK || opt.content_on_error) && local_file)
4867 {
4868 /* Bugfix: Prevent SIGSEGV when hstat.local_file was left NULL
4869 (i.e. due to opt.content_disposition). */
4870 if (hstat.local_file)
4871 {
4872 *local_file = hstat.local_file;
4873 hstat.local_file = NULL;
4874 }
4875 }
4876 free_hstat (&hstat);
4877
4878 return ret;
4879 }
4880
4881 /* Check whether the result of strptime() indicates success.
4882 strptime() returns the pointer to how far it got to in the string.
4883 The processing has been successful if the string is at `GMT' or
4884 `+X', or at the end of the string.
4885
4886 In extended regexp parlance, the function returns 1 if P matches
4887 "^ *(GMT|[+-][0-9]|$)", 0 otherwise. P being NULL (which strptime
4888 can return) is considered a failure and 0 is returned. */
4889 static bool
4890 check_end (const char *p)
4891 {
4892 if (!p)
4893 return false;
4894 while (c_isspace (*p))
4895 ++p;
4896 if (!*p
4897 || (p[0] == 'G' && p[1] == 'M' && p[2] == 'T')
4898 || ((p[0] == '+' || p[0] == '-') && c_isdigit (p[1])))
4899 return true;
4900 else
4901 return false;
4902 }
4903
4904 /* Convert the textual specification of time in TIME_STRING to the
4905 number of seconds since the Epoch.
4906
4907 TIME_STRING can be in any of the three formats RFC2616 allows the
4908 HTTP servers to emit -- RFC1123-date, RFC850-date or asctime-date,
4909 as well as the time format used in the Set-Cookie header.
4910 Timezones are ignored, and should be GMT.
4911
4912 Return the computed time_t representation, or -1 if the conversion
4913 fails.
4914
4915 This function uses strptime with various string formats for parsing
4916 TIME_STRING. This results in a parser that is not as lenient in
4917 interpreting TIME_STRING as I would like it to be. Being based on
4918 strptime, it always allows shortened months, one-digit days, etc.,
4919 but due to the multitude of formats in which time can be
4920 represented, an ideal HTTP time parser would be even more
4921 forgiving. It should completely ignore things like week days and
4922 concentrate only on the various forms of representing years,
4923 months, days, hours, minutes, and seconds. For example, it would
4924 be nice if it accepted ISO 8601 out of the box.
4925
4926 I've investigated free and PD code for this purpose, but none was
4927 usable. getdate was big and unwieldy, and had potential copyright
4928 issues, or so I was informed. Dr. Marcus Hennecke's atotm(),
4929 distributed with phttpd, is excellent, but we cannot use it because
4930 it is not assigned to the FSF. So I stuck it with strptime. */
4931
4932 time_t
4933 http_atotm (const char *time_string)
4934 {
4935 /* NOTE: Solaris strptime man page claims that %n and %t match white
4936 space, but that's not universally available. Instead, we simply
4937 use ` ' to mean "skip all WS", which works under all strptime
4938 implementations I've tested. */
4939
4940 static const char *time_formats[] = {
4941 "%a, %d %b %Y %T", /* rfc1123: Thu, 29 Jan 1998 22:12:57 */
4942 "%A, %d-%b-%y %T", /* rfc850: Thursday, 29-Jan-98 22:12:57 */
4943 "%a %b %d %T %Y", /* asctime: Thu Jan 29 22:12:57 1998 */
4944 "%a, %d-%b-%Y %T" /* cookies: Thu, 29-Jan-1998 22:12:57
4945 (used in Set-Cookie, defined in the
4946 Netscape cookie specification.) */
4947 };
4948 const char *oldlocale;
4949 char savedlocale[256];
4950 size_t i;
4951 time_t ret = (time_t) -1;
4952
4953 /* Solaris strptime fails to recognize English month names in
4954 non-English locales, which we work around by temporarily setting
4955 locale to C before invoking strptime. */
4956 oldlocale = setlocale (LC_TIME, NULL);
4957 if (oldlocale)
4958 {
4959 size_t l = strlen (oldlocale) + 1;
4960 if (l >= sizeof savedlocale)
4961 savedlocale[0] = '\0';
4962 else
4963 memcpy (savedlocale, oldlocale, l);
4964 }
4965 else savedlocale[0] = '\0';
4966
4967 setlocale (LC_TIME, "C");
4968
4969 for (i = 0; i < countof (time_formats); i++)
4970 {
4971 struct tm t;
4972
4973 /* Some versions of strptime use the existing contents of struct
4974 tm to recalculate the date according to format. Zero it out
4975 to prevent stack garbage from influencing strptime. */
4976 xzero (t);
4977
4978 if (check_end (strptime (time_string, time_formats[i], &t)))
4979 {
4980 ret = timegm (&t);
4981 break;
4982 }
4983 }
4984
4985 /* Restore the previous locale. */
4986 if (savedlocale[0])
4987 setlocale (LC_TIME, savedlocale);
4988
4989 return ret;
4990 }
4991
4992 /* Authorization support: We support three authorization schemes:
4993
4994 * `Basic' scheme, consisting of base64-ing USER:PASSWORD string;
4995
4996 * `Digest' scheme, added by Junio Hamano <[email protected]>,
4997 consisting of answering to the server's challenge with the proper
4998 MD5 digests.
4999
5000 * `NTLM' ("NT Lan Manager") scheme, based on code written by Daniel
5001 Stenberg for libcurl. Like digest, NTLM is based on a
5002 challenge-response mechanism, but unlike digest, it is non-standard
5003 (authenticates TCP connections rather than requests), undocumented
5004 and Microsoft-specific. */
5005
5006 /* Create the authentication header contents for the `Basic' scheme.
5007 This is done by encoding the string "USER:PASS" to base64 and
5008 prepending the string "Basic " in front of it. */
5009
5010 static char *
5011 basic_authentication_encode (const char *user, const char *passwd)
5012 {
5013 char buf_t1[256], buf_t2[256];
5014 char *t1, *t2, *ret;
5015 size_t len1 = strlen (user) + 1 + strlen (passwd);
5016
5017 if (len1 < sizeof (buf_t1))
5018 t1 = buf_t1;
5019 else
5020 t1 = xmalloc(len1 + 1);
5021
5022 if (BASE64_LENGTH (len1) < sizeof (buf_t2))
5023 t2 = buf_t2;
5024 else
5025 t2 = xmalloc (BASE64_LENGTH (len1) + 1);
5026
5027 sprintf (t1, "%s:%s", user, passwd);
5028 wget_base64_encode (t1, len1, t2);
5029
5030 ret = concat_strings ("Basic ", t2, (char *) 0);
5031
5032 if (t2 != buf_t2)
5033 xfree (t2);
5034
5035 if (t1 != buf_t1)
5036 xfree (t1);
5037
5038 return ret;
5039 }
5040
5041 #define SKIP_WS(x) do { \
5042 while (c_isspace (*(x))) \
5043 ++(x); \
5044 } while (0)
5045
5046 #ifdef ENABLE_DIGEST
5047 /* Dump the hexadecimal representation of HASH to BUF. HASH should be
5048 an array of 16 bytes containing the hash keys, and BUF should be a
5049 buffer of 33 writable characters (32 for hex digits plus one for
5050 zero termination). */
5051 static void
5052 dump_hash (char *buf, const unsigned char *hash)
5053 {
5054 int i;
5055
5056 for (i = 0; i < MD5_DIGEST_SIZE; i++, hash++)
5057 {
5058 *buf++ = XNUM_TO_digit (*hash >> 4);
5059 *buf++ = XNUM_TO_digit (*hash & 0xf);
5060 }
5061 *buf = '\0';
5062 }
5063
5064 /* Take the line apart to find the challenge, and compose a digest
5065 authorization header. See RFC2069 section 2.1.2. */
5066 static char *
5067 digest_authentication_encode (const char *au, const char *user,
5068 const char *passwd, const char *method,
5069 const char *path, uerr_t *auth_err)
5070 {
5071 static char *realm, *opaque, *nonce, *qop, *algorithm;
5072 static struct {
5073 const char *name;
5074 char **variable;
5075 } options[] = {
5076 { "realm", &realm },
5077 { "opaque", &opaque },
5078 { "nonce", &nonce },
5079 { "qop", &qop },
5080 { "algorithm", &algorithm }
5081 };
5082 char cnonce[16] = "";
5083 char *res = NULL;
5084 int res_len;
5085 size_t res_size;
5086 param_token name, value;
5087
5088
5089 realm = opaque = nonce = algorithm = qop = NULL;
5090
5091 au += 6; /* skip over `Digest' */
5092 while (extract_param (&au, &name, &value, ',', NULL))
5093 {
5094 size_t i;
5095 size_t namelen = name.e - name.b;
5096 for (i = 0; i < countof (options); i++)
5097 if (namelen == strlen (options[i].name)
5098 && 0 == strncmp (name.b, options[i].name,
5099 namelen))
5100 {
5101 *options[i].variable = strdupdelim (value.b, value.e);
5102 break;
5103 }
5104 }
5105
5106 if (qop && strcmp (qop, "auth"))
5107 {
5108 logprintf (LOG_NOTQUIET, _("Unsupported quality of protection '%s'.\n"), qop);
5109 xfree (qop); /* force freeing mem and continue */
5110 }
5111 else if (algorithm && strcmp (algorithm,"MD5") && strcmp (algorithm,"MD5-sess"))
5112 {
5113 logprintf (LOG_NOTQUIET, _("Unsupported algorithm '%s'.\n"), algorithm);
5114 xfree (algorithm); /* force freeing mem and continue */
5115 }
5116
5117 if (!realm || !nonce || !user || !passwd || !path || !method)
5118 {
5119 *auth_err = ATTRMISSING;
5120 goto cleanup;
5121 }
5122
5123 /* Calculate the digest value. */
5124 {
5125 struct md5_ctx ctx;
5126 unsigned char hash[MD5_DIGEST_SIZE];
5127 char a1buf[MD5_DIGEST_SIZE * 2 + 1], a2buf[MD5_DIGEST_SIZE * 2 + 1];
5128 char response_digest[MD5_DIGEST_SIZE * 2 + 1];
5129
5130 /* A1BUF = H(user ":" realm ":" password) */
5131 md5_init_ctx (&ctx);
5132 md5_process_bytes ((unsigned char *)user, strlen (user), &ctx);
5133 md5_process_bytes ((unsigned char *)":", 1, &ctx);
5134 md5_process_bytes ((unsigned char *)realm, strlen (realm), &ctx);
5135 md5_process_bytes ((unsigned char *)":", 1, &ctx);
5136 md5_process_bytes ((unsigned char *)passwd, strlen (passwd), &ctx);
5137 md5_finish_ctx (&ctx, hash);
5138
5139 dump_hash (a1buf, hash);
5140
5141 if (algorithm && !strcmp (algorithm, "MD5-sess"))
5142 {
5143 /* A1BUF = H( H(user ":" realm ":" password) ":" nonce ":" cnonce ) */
5144 snprintf (cnonce, sizeof (cnonce), "%08x",
5145 (unsigned) random_number (INT_MAX));
5146
5147 md5_init_ctx (&ctx);
5148 /* md5_process_bytes (hash, MD5_DIGEST_SIZE, &ctx); */
5149 md5_process_bytes (a1buf, MD5_DIGEST_SIZE * 2, &ctx);
5150 md5_process_bytes ((unsigned char *)":", 1, &ctx);
5151 md5_process_bytes ((unsigned char *)nonce, strlen (nonce), &ctx);
5152 md5_process_bytes ((unsigned char *)":", 1, &ctx);
5153 md5_process_bytes ((unsigned char *)cnonce, strlen (cnonce), &ctx);
5154 md5_finish_ctx (&ctx, hash);
5155
5156 dump_hash (a1buf, hash);
5157 }
5158
5159 /* A2BUF = H(method ":" path) */
5160 md5_init_ctx (&ctx);
5161 md5_process_bytes ((unsigned char *)method, strlen (method), &ctx);
5162 md5_process_bytes ((unsigned char *)":", 1, &ctx);
5163 md5_process_bytes ((unsigned char *)path, strlen (path), &ctx);
5164 md5_finish_ctx (&ctx, hash);
5165 dump_hash (a2buf, hash);
5166
5167 if (qop && !strcmp (qop, "auth"))
5168 {
5169 /* RFC 2617 Digest Access Authentication */
5170 /* generate random hex string */
5171 if (!*cnonce)
5172 snprintf (cnonce, sizeof (cnonce), "%08x",
5173 (unsigned) random_number (INT_MAX));
5174
5175 /* RESPONSE_DIGEST = H(A1BUF ":" nonce ":" noncecount ":" clientnonce ":" qop ": " A2BUF) */
5176 md5_init_ctx (&ctx);
5177 md5_process_bytes ((unsigned char *)a1buf, MD5_DIGEST_SIZE * 2, &ctx);
5178 md5_process_bytes ((unsigned char *)":", 1, &ctx);
5179 md5_process_bytes ((unsigned char *)nonce, strlen (nonce), &ctx);
5180 md5_process_bytes ((unsigned char *)":", 1, &ctx);
5181 md5_process_bytes ((unsigned char *)"00000001", 8, &ctx); /* TODO: keep track of server nonce values */
5182 md5_process_bytes ((unsigned char *)":", 1, &ctx);
5183 md5_process_bytes ((unsigned char *)cnonce, strlen (cnonce), &ctx);
5184 md5_process_bytes ((unsigned char *)":", 1, &ctx);
5185 md5_process_bytes ((unsigned char *)qop, strlen (qop), &ctx);
5186 md5_process_bytes ((unsigned char *)":", 1, &ctx);
5187 md5_process_bytes ((unsigned char *)a2buf, MD5_DIGEST_SIZE * 2, &ctx);
5188 md5_finish_ctx (&ctx, hash);
5189 }
5190 else
5191 {
5192 /* RFC 2069 Digest Access Authentication */
5193 /* RESPONSE_DIGEST = H(A1BUF ":" nonce ":" A2BUF) */
5194 md5_init_ctx (&ctx);
5195 md5_process_bytes ((unsigned char *)a1buf, MD5_DIGEST_SIZE * 2, &ctx);
5196 md5_process_bytes ((unsigned char *)":", 1, &ctx);
5197 md5_process_bytes ((unsigned char *)nonce, strlen (nonce), &ctx);
5198 md5_process_bytes ((unsigned char *)":", 1, &ctx);
5199 md5_process_bytes ((unsigned char *)a2buf, MD5_DIGEST_SIZE * 2, &ctx);
5200 md5_finish_ctx (&ctx, hash);
5201 }
5202
5203 dump_hash (response_digest, hash);
5204
5205 res_size = strlen (user)
5206 + strlen (realm)
5207 + strlen (nonce)
5208 + strlen (path)
5209 + 2 * MD5_DIGEST_SIZE /*strlen (response_digest)*/
5210 + (opaque ? strlen (opaque) : 0)
5211 + (algorithm ? strlen (algorithm) : 0)
5212 + (qop ? 128: 0)
5213 + strlen (cnonce)
5214 + 128;
5215
5216 res = xmalloc (res_size);
5217
5218 if (qop && !strcmp (qop, "auth"))
5219 {
5220 res_len = snprintf (res, res_size, "Digest "\
5221 "username=\"%s\", realm=\"%s\", nonce=\"%s\", uri=\"%s\", response=\"%s\""\
5222 ", qop=auth, nc=00000001, cnonce=\"%s\"",
5223 user, realm, nonce, path, response_digest, cnonce);
5224
5225 }
5226 else
5227 {
5228 res_len = snprintf (res, res_size, "Digest "\
5229 "username=\"%s\", realm=\"%s\", nonce=\"%s\", uri=\"%s\", response=\"%s\"",
5230 user, realm, nonce, path, response_digest);
5231 }
5232
5233 if (opaque)
5234 {
5235 res_len += snprintf (res + res_len, res_size - res_len, ", opaque=\"%s\"", opaque);
5236 }
5237
5238 if (algorithm)
5239 {
5240 snprintf (res + res_len, res_size - res_len, ", algorithm=\"%s\"", algorithm);
5241 }
5242 }
5243
5244 cleanup:
5245 xfree (realm);
5246 xfree (opaque);
5247 xfree (nonce);
5248 xfree (qop);
5249 xfree (algorithm);
5250
5251 return res;
5252 }
5253 #endif /* ENABLE_DIGEST */
5254
5255 /* Computing the size of a string literal must take into account that
5256 value returned by sizeof includes the terminating \0. */
5257 #define STRSIZE(literal) (sizeof (literal) - 1)
5258
5259 /* Whether chars in [b, e) begin with the literal string provided as
5260 first argument and are followed by whitespace or terminating \0.
5261 The comparison is case-insensitive. */
5262 #define STARTS(literal, b, e) \
5263 ((e > b) \
5264 && ((size_t) ((e) - (b))) >= STRSIZE (literal) \
5265 && 0 == c_strncasecmp (b, literal, STRSIZE (literal)) \
5266 && ((size_t) ((e) - (b)) == STRSIZE (literal) \
5267 || c_isspace (b[STRSIZE (literal)])))
5268
5269 static bool
5270 known_authentication_scheme_p (const char *hdrbeg, const char *hdrend)
5271 {
5272 return STARTS ("Basic", hdrbeg, hdrend)
5273 #ifdef ENABLE_DIGEST
5274 || STARTS ("Digest", hdrbeg, hdrend)
5275 #endif
5276 #ifdef ENABLE_NTLM
5277 || STARTS ("NTLM", hdrbeg, hdrend)
5278 #endif
5279 ;
5280 }
5281
5282 #undef STARTS
5283
5284 /* Create the HTTP authorization request header. When the
5285 `WWW-Authenticate' response header is seen, according to the
5286 authorization scheme specified in that header (`Basic' and `Digest'
5287 are supported by the current implementation), produce an
5288 appropriate HTTP authorization request header. */
5289 static char *
5290 create_authorization_line (const char *au, const char *user,
5291 const char *passwd, const char *method,
5292 const char *path, bool *finished, uerr_t *auth_err)
5293 {
5294 /* We are called only with known schemes, so we can dispatch on the
5295 first letter. */
5296 switch (c_toupper (*au))
5297 {
5298 case 'B': /* Basic */
5299 *finished = true;
5300 return basic_authentication_encode (user, passwd);
5301 #ifdef ENABLE_DIGEST
5302 case 'D': /* Digest */
5303 *finished = true;
5304 return digest_authentication_encode (au, user, passwd, method, path, auth_err);
5305 #endif
5306 #ifdef ENABLE_NTLM
5307 case 'N': /* NTLM */
5308 if (!ntlm_input (&pconn.ntlm, au))
5309 {
5310 *finished = true;
5311 return NULL;
5312 }
5313 return ntlm_output (&pconn.ntlm, user, passwd, finished);
5314 #endif
5315 default:
5316 /* We shouldn't get here -- this function should be only called
5317 with values approved by known_authentication_scheme_p. */
5318 abort ();
5319 }
5320 }
5321
5322 static void
5323 load_cookies (void)
5324 {
5325 if (!wget_cookie_jar)
5326 wget_cookie_jar = cookie_jar_new ();
5327 if (opt.cookies_input && !cookies_loaded_p)
5328 {
5329 cookie_jar_load (wget_cookie_jar, opt.cookies_input);
5330 cookies_loaded_p = true;
5331 }
5332 }
5333
5334 void
5335 save_cookies (void)
5336 {
5337 if (wget_cookie_jar)
5338 cookie_jar_save (wget_cookie_jar, opt.cookies_output);
5339 }
5340
5341 #if defined DEBUG_MALLOC || defined TESTING
5342 void
5343 http_cleanup (void)
5344 {
5345 if (pconn_active)
5346 invalidate_persistent ();
5347
5348 if (wget_cookie_jar)
5349 {
5350 cookie_jar_delete (wget_cookie_jar);
5351 wget_cookie_jar = NULL;
5352 }
5353
5354 if (basic_authed_hosts)
5355 {
5356 hash_table_iterator iter;
5357 for (hash_table_iterate (basic_authed_hosts, &iter); hash_table_iter_next (&iter); )
5358 {
5359 xfree (iter.key);
5360 }
5361 hash_table_destroy (basic_authed_hosts);
5362 basic_authed_hosts = NULL;
5363 }
5364 }
5365 #endif
5366
5367 void
5368 ensure_extension (struct http_stat *hs, const char *ext, int *dt)
5369 {
5370 char *last_period_in_local_filename = strrchr (hs->local_file, '.');
5371 char shortext[8];
5372 int len;
5373 shortext[0] = '\0';
5374 len = strlen (ext);
5375 if (len == 5)
5376 {
5377 memcpy (shortext, ext, len - 1);
5378 shortext[len - 1] = '\0';
5379 }
5380
5381 if (last_period_in_local_filename == NULL
5382 || !(0 == strcasecmp (last_period_in_local_filename, shortext)
5383 || 0 == strcasecmp (last_period_in_local_filename, ext)))
5384 {
5385 int local_filename_len = strlen (hs->local_file);
5386 /* Resize the local file, allowing for ".html" preceded by
5387 optional ".NUMBER". */
5388 hs->local_file = xrealloc (hs->local_file,
5389 local_filename_len + 24 + len);
5390 strcpy (hs->local_file + local_filename_len, ext);
5391 /* If clobbering is not allowed and the file, as named,
5392 exists, tack on ".NUMBER.html" instead. */
5393 if (!ALLOW_CLOBBER && file_exists_p (hs->local_file, NULL))
5394 {
5395 int ext_num = 1;
5396 do
5397 sprintf (hs->local_file + local_filename_len,
5398 ".%d%s", ext_num++, ext);
5399 while (file_exists_p (hs->local_file, NULL));
5400 }
5401 *dt |= ADDED_HTML_EXTENSION;
5402 }
5403 }
5404
5405 #ifdef TESTING
5406
5407 const char *
5408 test_parse_range_header (void)
5409 {
5410 unsigned i;
5411 static const struct {
5412 const char * rangehdr;
5413 const wgint firstbyte;
5414 const wgint lastbyte;
5415 const wgint length;
5416 const bool shouldPass;
5417 } test_array[] = {
5418 { "bytes 0-1000/1000", 0, 1000, 1000, false },
5419 { "bytes 0-999/1000", 0, 999, 1000, true },
5420 { "bytes 100-99/1000", 100, 99, 1000, false },
5421 { "bytes 100-100/1000", 100, 100, 1000, true },
5422 { "bytes 0-1000/100000000", 0, 1000, 100000000, true },
5423 { "bytes 1-999/1000", 1, 999, 1000, true },
5424 { "bytes 42-1233/1234", 42, 1233, 1234, true },
5425 { "bytes 42-1233/*", 42, 1233, -1, true },
5426 { "bytes 0-2147483648/2147483649", 0, 2147483648U, 2147483649U, true },
5427 { "bytes 2147483648-4294967296/4294967297", 2147483648U, 4294967296ULL, 4294967297ULL, true },
5428 };
5429
5430 wgint firstbyteptr[sizeof(wgint)];
5431 wgint lastbyteptr[sizeof(wgint)];
5432 wgint lengthptr[sizeof(wgint)];
5433 bool result;
5434 for (i = 0; i < countof (test_array); i++)
5435 {
5436 result = parse_content_range (test_array[i].rangehdr, firstbyteptr, lastbyteptr, lengthptr);
5437 #if 0
5438 printf ("%ld %ld\n", test_array[i].firstbyte, *firstbyteptr);
5439 printf ("%ld %ld\n", test_array[i].lastbyte, *lastbyteptr);
5440 printf ("%ld %ld\n", test_array[i].length, *lengthptr);
5441 printf ("\n");
5442 #endif
5443 mu_assert ("test_parse_range_header: False Negative", result == test_array[i].shouldPass);
5444 mu_assert ("test_parse_range_header: Bad parse", test_array[i].firstbyte == *firstbyteptr &&
5445 test_array[i].lastbyte == *lastbyteptr &&
5446 test_array[i].length == *lengthptr);
5447 }
5448
5449 return NULL;
5450 }
5451
5452 const char *
5453 test_parse_content_disposition (void)
5454 {
5455 unsigned i;
5456 static const struct {
5457 const char *hdrval;
5458 const char *filename;
5459 bool result;
5460 } test_array[] = {
5461 { "filename=\"file.ext\"", "file.ext", true },
5462 { "attachment; filename=\"file.ext\"", "file.ext", true },
5463 { "attachment; filename=\"file.ext\"; dummy", "file.ext", true },
5464 { "attachment", NULL, false },
5465 { "attachment; filename*=UTF-8'en-US'hello.txt", "hello.txt", true },
5466 { "attachment; filename*0=\"hello\"; filename*1=\"world.txt\"",
5467 "helloworld.txt", true },
5468 { "attachment; filename=\"A.ext\"; filename*=\"B.ext\"", "B.ext", true },
5469 { "attachment; filename*=\"A.ext\"; filename*0=\"B\"; filename*1=\"B.ext\"",
5470 "A.ext", true },
5471 { "filename**0=\"A\"; filename**1=\"A.ext\"; filename*0=\"B\";\
5472 filename*1=\"B\"", "AA.ext", true },
5473 };
5474
5475 for (i = 0; i < countof (test_array); ++i)
5476 {
5477 char *filename;
5478 bool res;
5479
5480 res = parse_content_disposition (test_array[i].hdrval, &filename);
5481
5482 mu_assert ("test_parse_content_disposition: wrong result",
5483 res == test_array[i].result
5484 && (res == false
5485 || 0 == strcmp (test_array[i].filename, filename)));
5486 xfree (filename);
5487 }
5488
5489 return NULL;
5490 }
5491
5492 #endif /* TESTING */
5493
5494 /*
5495 * vim: et sts=2 sw=2 cino+={s
5496 */