| 1 |
/* Handling of recursive HTTP retrieving. |
| 2 |
Copyright (C) 1996-2012, 2015, 2018-2024 Free Software Foundation, |
| 3 |
Inc. |
| 4 |
|
| 5 |
This file is part of GNU Wget. |
| 6 |
|
| 7 |
GNU Wget is free software; you can redistribute it and/or modify |
| 8 |
it under the terms of the GNU General Public License as published by |
| 9 |
the Free Software Foundation; either version 3 of the License, or |
| 10 |
(at your option) any later version. |
| 11 |
|
| 12 |
GNU Wget is distributed in the hope that it will be useful, |
| 13 |
but WITHOUT ANY WARRANTY; without even the implied warranty of |
| 14 |
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| 15 |
GNU General Public License for more details. |
| 16 |
|
| 17 |
You should have received a copy of the GNU General Public License |
| 18 |
along with Wget. If not, see <http://www.gnu.org/licenses/>. |
| 19 |
|
| 20 |
Additional permission under GNU GPL version 3 section 7 |
| 21 |
|
| 22 |
If you modify this program, or any covered work, by linking or |
| 23 |
combining it with the OpenSSL project's OpenSSL library (or a |
| 24 |
modified version of that library), containing parts covered by the |
| 25 |
terms of the OpenSSL or SSLeay licenses, the Free Software Foundation |
| 26 |
grants you additional permission to convey the resulting work. |
| 27 |
Corresponding Source for a non-source form of such a combination |
| 28 |
shall include the source code for the parts of OpenSSL used as well |
| 29 |
as that of the covered work. */ |
| 30 |
|
| 31 |
#include "wget.h" |
| 32 |
|
| 33 |
#include <stdio.h> |
| 34 |
#include <stdlib.h> |
| 35 |
#include <string.h> |
| 36 |
#include <unistd.h> |
| 37 |
#include <errno.h> |
| 38 |
#include <assert.h> |
| 39 |
|
| 40 |
#include "url.h" |
| 41 |
#include "recur.h" |
| 42 |
#include "utils.h" |
| 43 |
#include "retr.h" |
| 44 |
#include "ftp.h" |
| 45 |
#include "host.h" |
| 46 |
#include "hash.h" |
| 47 |
#include "res.h" |
| 48 |
#include "convert.h" |
| 49 |
#include "html-url.h" |
| 50 |
#include "css-url.h" |
| 51 |
#include "spider.h" |
| 52 |
#include "exits.h" |
| 53 |
|
| 54 |
/* Functions for maintaining the URL queue. */ |
| 55 |
|
| 56 |
struct queue_element { |
| 57 |
const char *url; /* the URL to download */ |
| 58 |
const char *referer; /* the referring document */ |
| 59 |
int depth; /* the depth */ |
| 60 |
bool html_allowed; /* whether the document is allowed to |
| 61 |
be treated as HTML. */ |
| 62 |
struct iri *iri; /* sXXXav */ |
| 63 |
bool css_allowed; /* whether the document is allowed to |
| 64 |
be treated as CSS. */ |
| 65 |
struct queue_element *next; /* next element in queue */ |
| 66 |
}; |
| 67 |
|
| 68 |
struct url_queue { |
| 69 |
struct queue_element *head; |
| 70 |
struct queue_element *tail; |
| 71 |
int count, maxcount; |
| 72 |
}; |
| 73 |
|
| 74 |
/* Create a URL queue. */ |
| 75 |
|
| 76 |
static struct url_queue * |
| 77 |
url_queue_new (void) |
| 78 |
{ |
| 79 |
struct url_queue *queue = xnew0 (struct url_queue); |
| 80 |
return queue; |
| 81 |
} |
| 82 |
|
| 83 |
/* Delete a URL queue. */ |
| 84 |
|
| 85 |
static void |
| 86 |
url_queue_delete (struct url_queue *queue) |
| 87 |
{ |
| 88 |
xfree (queue); |
| 89 |
} |
| 90 |
|
| 91 |
/* Enqueue a URL in the queue. The queue is FIFO: the items will be |
| 92 |
retrieved ("dequeued") from the queue in the order they were placed |
| 93 |
into it. */ |
| 94 |
|
| 95 |
static void |
| 96 |
url_enqueue (struct url_queue *queue, struct iri *i, |
| 97 |
const char *url, const char *referer, int depth, |
| 98 |
bool html_allowed, bool css_allowed) |
| 99 |
{ |
| 100 |
struct queue_element *qel = xnew (struct queue_element); |
| 101 |
qel->iri = i; |
| 102 |
qel->url = url; |
| 103 |
qel->referer = referer; |
| 104 |
qel->depth = depth; |
| 105 |
qel->html_allowed = html_allowed; |
| 106 |
qel->css_allowed = css_allowed; |
| 107 |
qel->next = NULL; |
| 108 |
|
| 109 |
++queue->count; |
| 110 |
if (queue->count > queue->maxcount) |
| 111 |
queue->maxcount = queue->count; |
| 112 |
|
| 113 |
DEBUGP (("Enqueuing %s at depth %d\n", |
| 114 |
quotearg_n_style (0, escape_quoting_style, url), depth)); |
| 115 |
DEBUGP (("Queue count %d, maxcount %d.\n", queue->count, queue->maxcount)); |
| 116 |
|
| 117 |
if (i) |
| 118 |
DEBUGP (("[IRI Enqueuing %s with %s\n", quote_n (0, url), |
| 119 |
i->uri_encoding ? quote_n (1, i->uri_encoding) : "None")); |
| 120 |
|
| 121 |
if (queue->tail) |
| 122 |
queue->tail->next = qel; |
| 123 |
queue->tail = qel; |
| 124 |
|
| 125 |
if (!queue->head) |
| 126 |
queue->head = queue->tail; |
| 127 |
} |
| 128 |
|
| 129 |
/* Take a URL out of the queue. Return true if this operation |
| 130 |
succeeded, or false if the queue is empty. */ |
| 131 |
|
| 132 |
static bool |
| 133 |
url_dequeue (struct url_queue *queue, struct iri **i, |
| 134 |
const char **url, const char **referer, int *depth, |
| 135 |
bool *html_allowed, bool *css_allowed) |
| 136 |
{ |
| 137 |
struct queue_element *qel = queue->head; |
| 138 |
|
| 139 |
if (!qel) |
| 140 |
return false; |
| 141 |
|
| 142 |
queue->head = queue->head->next; |
| 143 |
if (!queue->head) |
| 144 |
queue->tail = NULL; |
| 145 |
|
| 146 |
*i = qel->iri; |
| 147 |
*url = qel->url; |
| 148 |
*referer = qel->referer; |
| 149 |
*depth = qel->depth; |
| 150 |
*html_allowed = qel->html_allowed; |
| 151 |
*css_allowed = qel->css_allowed; |
| 152 |
|
| 153 |
--queue->count; |
| 154 |
|
| 155 |
DEBUGP (("Dequeuing %s at depth %d\n", |
| 156 |
quotearg_n_style (0, escape_quoting_style, qel->url), qel->depth)); |
| 157 |
DEBUGP (("Queue count %d, maxcount %d.\n", queue->count, queue->maxcount)); |
| 158 |
|
| 159 |
xfree (qel); |
| 160 |
return true; |
| 161 |
} |
| 162 |
|
| 163 |
static void blacklist_add (struct hash_table *blacklist, const char *url) |
| 164 |
{ |
| 165 |
char *url_unescaped = xstrdup (url); |
| 166 |
|
| 167 |
url_unescape (url_unescaped); |
| 168 |
string_set_add (blacklist, url_unescaped); |
| 169 |
xfree (url_unescaped); |
| 170 |
} |
| 171 |
|
| 172 |
static int blacklist_contains (struct hash_table *blacklist, const char *url) |
| 173 |
{ |
| 174 |
char *url_unescaped = xstrdup(url); |
| 175 |
int ret; |
| 176 |
|
| 177 |
url_unescape (url_unescaped); |
| 178 |
ret = string_set_contains (blacklist, url_unescaped); |
| 179 |
xfree (url_unescaped); |
| 180 |
|
| 181 |
return ret; |
| 182 |
} |
| 183 |
|
| 184 |
typedef enum |
| 185 |
{ |
| 186 |
WG_RR_SUCCESS, WG_RR_BLACKLIST, WG_RR_NOTHTTPS, WG_RR_NONHTTP, WG_RR_ABSOLUTE, |
| 187 |
WG_RR_DOMAIN, WG_RR_PARENT, WG_RR_LIST, WG_RR_REGEX, WG_RR_RULES, |
| 188 |
WG_RR_SPANNEDHOST, WG_RR_ROBOTS |
| 189 |
} reject_reason; |
| 190 |
|
| 191 |
static reject_reason download_child (const struct urlpos *, struct url *, int, |
| 192 |
struct url *, struct hash_table *, struct iri *); |
| 193 |
static reject_reason descend_redirect (const char *, struct url *, int, |
| 194 |
struct url *, struct hash_table *, struct iri *); |
| 195 |
static void write_reject_log_header (FILE *); |
| 196 |
static void write_reject_log_reason (FILE *, reject_reason, |
| 197 |
const struct url *, const struct url *); |
| 198 |
|
| 199 |
/* Retrieve a part of the web beginning with START_URL. This used to |
| 200 |
be called "recursive retrieval", because the old function was |
| 201 |
recursive and implemented depth-first search. retrieve_tree on the |
| 202 |
other hand implements breadth-search traversal of the tree, which |
| 203 |
results in much nicer ordering of downloads. |
| 204 |
|
| 205 |
The algorithm this function uses is simple: |
| 206 |
|
| 207 |
1. put START_URL in the queue. |
| 208 |
2. while there are URLs in the queue: |
| 209 |
|
| 210 |
3. get next URL from the queue. |
| 211 |
4. download it. |
| 212 |
5. if the URL is HTML and its depth does not exceed maximum depth, |
| 213 |
get the list of URLs embedded therein. |
| 214 |
6. for each of those URLs do the following: |
| 215 |
|
| 216 |
7. if the URL is not one of those downloaded before, and if it |
| 217 |
satisfies the criteria specified by the various command-line |
| 218 |
options, add it to the queue. */ |
| 219 |
|
| 220 |
uerr_t |
| 221 |
retrieve_tree (struct url *start_url_parsed, struct iri *pi) |
| 222 |
{ |
| 223 |
uerr_t status = RETROK; |
| 224 |
|
| 225 |
/* The queue of URLs we need to load. */ |
| 226 |
struct url_queue *queue; |
| 227 |
|
| 228 |
/* The URLs we do not wish to enqueue, because they are already in |
| 229 |
the queue, but haven't been downloaded yet. */ |
| 230 |
struct hash_table *blacklist; |
| 231 |
|
| 232 |
struct iri *i = iri_new (); |
| 233 |
|
| 234 |
FILE *rejectedlog = NULL; /* Don't write a rejected log. */ |
| 235 |
|
| 236 |
/* Duplicate pi struct if not NULL */ |
| 237 |
if (pi) |
| 238 |
{ |
| 239 |
#define COPYSTR(x) (x) ? xstrdup(x) : NULL; |
| 240 |
i->uri_encoding = COPYSTR (pi->uri_encoding); |
| 241 |
i->content_encoding = COPYSTR (pi->content_encoding); |
| 242 |
i->utf8_encode = pi->utf8_encode; |
| 243 |
#undef COPYSTR |
| 244 |
} |
| 245 |
#ifdef ENABLE_IRI |
| 246 |
else |
| 247 |
set_uri_encoding (i, opt.locale, true); |
| 248 |
#endif |
| 249 |
|
| 250 |
queue = url_queue_new (); |
| 251 |
blacklist = make_string_hash_table (0); |
| 252 |
|
| 253 |
/* Enqueue the starting URL. Use start_url_parsed->url rather than |
| 254 |
just URL so we enqueue the canonical form of the URL. */ |
| 255 |
url_enqueue (queue, i, xstrdup (start_url_parsed->url), NULL, 0, true, |
| 256 |
false); |
| 257 |
blacklist_add (blacklist, start_url_parsed->url); |
| 258 |
|
| 259 |
if (opt.rejected_log) |
| 260 |
{ |
| 261 |
rejectedlog = fopen (opt.rejected_log, "w"); |
| 262 |
write_reject_log_header (rejectedlog); |
| 263 |
if (!rejectedlog) |
| 264 |
logprintf (LOG_NOTQUIET, "%s: %s\n", opt.rejected_log, strerror (errno)); |
| 265 |
} |
| 266 |
|
| 267 |
while (1) |
| 268 |
{ |
| 269 |
bool descend = false; |
| 270 |
char *url, *referer, *file = NULL; |
| 271 |
int depth; |
| 272 |
bool html_allowed, css_allowed; |
| 273 |
bool is_css = false; |
| 274 |
bool dash_p_leaf_HTML = false; |
| 275 |
|
| 276 |
if (opt.quota && total_downloaded_bytes > opt.quota) |
| 277 |
break; |
| 278 |
if (status == FWRITEERR) |
| 279 |
break; |
| 280 |
|
| 281 |
/* Get the next URL from the queue... */ |
| 282 |
|
| 283 |
if (!url_dequeue (queue, (struct iri **) &i, |
| 284 |
(const char **)&url, (const char **)&referer, |
| 285 |
&depth, &html_allowed, &css_allowed)) |
| 286 |
break; |
| 287 |
|
| 288 |
/* ...and download it. Note that this download is in most cases |
| 289 |
unconditional, as download_child already makes sure a file |
| 290 |
doesn't get enqueued twice -- and yet this check is here, and |
| 291 |
not in download_child. This is so that if you run `wget -r |
| 292 |
URL1 URL2', and a random URL is encountered once under URL1 |
| 293 |
and again under URL2, but at a different (possibly smaller) |
| 294 |
depth, we want the URL's children to be taken into account |
| 295 |
the second time. */ |
| 296 |
if (dl_url_file_map && hash_table_contains (dl_url_file_map, url)) |
| 297 |
{ |
| 298 |
bool is_css_bool; |
| 299 |
|
| 300 |
file = xstrdup (hash_table_get (dl_url_file_map, url)); |
| 301 |
|
| 302 |
DEBUGP (("Already downloaded \"%s\", reusing it from \"%s\".\n", |
| 303 |
url, file)); |
| 304 |
|
| 305 |
if ((is_css_bool = (css_allowed |
| 306 |
&& downloaded_css_set |
| 307 |
&& string_set_contains (downloaded_css_set, file))) |
| 308 |
|| (html_allowed |
| 309 |
&& downloaded_html_set |
| 310 |
&& string_set_contains (downloaded_html_set, file))) |
| 311 |
{ |
| 312 |
descend = true; |
| 313 |
is_css = is_css_bool; |
| 314 |
} |
| 315 |
} |
| 316 |
else |
| 317 |
{ |
| 318 |
int dt = 0, url_err; |
| 319 |
char *redirected = NULL; |
| 320 |
struct url *url_parsed = url_parse (url, &url_err, i, true); |
| 321 |
|
| 322 |
if (!url_parsed) |
| 323 |
{ |
| 324 |
logprintf (LOG_NOTQUIET, "%s: %s.\n",url, url_error (url_err)); |
| 325 |
inform_exit_status (URLERROR); |
| 326 |
} |
| 327 |
else |
| 328 |
{ |
| 329 |
|
| 330 |
status = retrieve_url (url_parsed, url, &file, &redirected, referer, |
| 331 |
&dt, false, i, true); |
| 332 |
|
| 333 |
if (html_allowed && file && status == RETROK |
| 334 |
&& (dt & RETROKF) && (dt & TEXTHTML)) |
| 335 |
{ |
| 336 |
descend = true; |
| 337 |
is_css = false; |
| 338 |
} |
| 339 |
|
| 340 |
/* a little different, css_allowed can override content type |
| 341 |
lots of web servers serve css with an incorrect content type |
| 342 |
*/ |
| 343 |
if (file && status == RETROK |
| 344 |
&& (dt & RETROKF) && |
| 345 |
((dt & TEXTCSS) || css_allowed)) |
| 346 |
{ |
| 347 |
descend = true; |
| 348 |
is_css = true; |
| 349 |
} |
| 350 |
|
| 351 |
if (redirected) |
| 352 |
{ |
| 353 |
/* We have been redirected, possibly to another host, or |
| 354 |
different path, or wherever. Check whether we really |
| 355 |
want to follow it. */ |
| 356 |
if (descend) |
| 357 |
{ |
| 358 |
reject_reason r = descend_redirect (redirected, url_parsed, |
| 359 |
depth, start_url_parsed, blacklist, i); |
| 360 |
if (r == WG_RR_SUCCESS) |
| 361 |
{ |
| 362 |
/* Make sure that the old pre-redirect form gets |
| 363 |
blacklisted. */ |
| 364 |
blacklist_add (blacklist, url); |
| 365 |
} |
| 366 |
else |
| 367 |
{ |
| 368 |
write_reject_log_reason (rejectedlog, r, url_parsed, start_url_parsed); |
| 369 |
descend = false; |
| 370 |
} |
| 371 |
} |
| 372 |
|
| 373 |
xfree (url); |
| 374 |
url = redirected; |
| 375 |
} |
| 376 |
else |
| 377 |
{ |
| 378 |
xfree (url); |
| 379 |
url = xstrdup (url_parsed->url); |
| 380 |
} |
| 381 |
url_free (url_parsed); |
| 382 |
} |
| 383 |
} |
| 384 |
|
| 385 |
if (opt.spider) |
| 386 |
{ |
| 387 |
visited_url (url, referer); |
| 388 |
} |
| 389 |
|
| 390 |
if (descend |
| 391 |
&& depth >= opt.reclevel && opt.reclevel != INFINITE_RECURSION) |
| 392 |
{ |
| 393 |
if (opt.page_requisites |
| 394 |
&& (depth == opt.reclevel || depth == opt.reclevel + 1)) |
| 395 |
{ |
| 396 |
/* When -p is specified, we are allowed to exceed the |
| 397 |
maximum depth, but only for the "inline" links, |
| 398 |
i.e. those that are needed to display the page. |
| 399 |
Originally this could exceed the depth at most by |
| 400 |
one, but we allow one more level so that the leaf |
| 401 |
pages that contain frames can be loaded |
| 402 |
correctly. */ |
| 403 |
dash_p_leaf_HTML = true; |
| 404 |
} |
| 405 |
else |
| 406 |
{ |
| 407 |
/* Either -p wasn't specified or it was and we've |
| 408 |
already spent the two extra (pseudo-)levels that it |
| 409 |
affords us, so we need to bail out. */ |
| 410 |
DEBUGP (("Not descending further; at depth %d, max. %d.\n", |
| 411 |
depth, opt.reclevel)); |
| 412 |
descend = false; |
| 413 |
} |
| 414 |
} |
| 415 |
|
| 416 |
/* If the downloaded document was HTML or CSS, parse it and enqueue the |
| 417 |
links it contains. */ |
| 418 |
|
| 419 |
if (descend) |
| 420 |
{ |
| 421 |
bool meta_disallow_follow = false; |
| 422 |
struct urlpos *children |
| 423 |
= is_css ? get_urls_css_file (file, url) : |
| 424 |
get_urls_html (file, url, &meta_disallow_follow, i); |
| 425 |
|
| 426 |
if (opt.use_robots && meta_disallow_follow) |
| 427 |
{ |
| 428 |
logprintf(LOG_VERBOSE, _("nofollow attribute found in %s. Will not follow any links on this page\n"), file); |
| 429 |
free_urlpos (children); |
| 430 |
children = NULL; |
| 431 |
} |
| 432 |
|
| 433 |
if (children) |
| 434 |
{ |
| 435 |
struct urlpos *child = children; |
| 436 |
struct url *url_parsed = url_parse (url, NULL, i, true); |
| 437 |
struct iri *ci; |
| 438 |
char *referer_url = url; |
| 439 |
bool strip_auth; |
| 440 |
|
| 441 |
assert (url_parsed != NULL); |
| 442 |
|
| 443 |
if (!url_parsed) |
| 444 |
continue; |
| 445 |
|
| 446 |
strip_auth = (url_parsed && url_parsed->user); |
| 447 |
|
| 448 |
/* Strip auth info if present */ |
| 449 |
if (strip_auth) |
| 450 |
referer_url = url_string (url_parsed, URL_AUTH_HIDE); |
| 451 |
|
| 452 |
for (; child; child = child->next) |
| 453 |
{ |
| 454 |
reject_reason r; |
| 455 |
|
| 456 |
if (child->ignore_when_downloading) |
| 457 |
{ |
| 458 |
DEBUGP (("Not following due to 'ignore' flag: %s\n", child->url->url)); |
| 459 |
continue; |
| 460 |
} |
| 461 |
|
| 462 |
if (dash_p_leaf_HTML && !child->link_inline_p) |
| 463 |
{ |
| 464 |
DEBUGP (("Not following due to 'link inline' flag: %s\n", child->url->url)); |
| 465 |
continue; |
| 466 |
} |
| 467 |
|
| 468 |
r = download_child (child, url_parsed, depth, |
| 469 |
start_url_parsed, blacklist, i); |
| 470 |
if (r == WG_RR_SUCCESS) |
| 471 |
{ |
| 472 |
ci = iri_new (); |
| 473 |
set_uri_encoding (ci, i->content_encoding, false); |
| 474 |
url_enqueue (queue, ci, xstrdup (child->url->url), |
| 475 |
xstrdup (referer_url), depth + 1, |
| 476 |
child->link_expect_html, |
| 477 |
child->link_expect_css); |
| 478 |
/* We blacklist the URL we have enqueued, because we |
| 479 |
don't want to enqueue (and hence download) the |
| 480 |
same URL twice. */ |
| 481 |
blacklist_add (blacklist, child->url->url); |
| 482 |
} |
| 483 |
else |
| 484 |
{ |
| 485 |
write_reject_log_reason (rejectedlog, r, child->url, url_parsed); |
| 486 |
} |
| 487 |
} |
| 488 |
|
| 489 |
if (strip_auth) |
| 490 |
xfree (referer_url); |
| 491 |
url_free (url_parsed); |
| 492 |
free_urlpos (children); |
| 493 |
} |
| 494 |
} |
| 495 |
|
| 496 |
if (file |
| 497 |
&& (opt.delete_after |
| 498 |
|| opt.spider /* opt.recursive is implicitly true */ |
| 499 |
|| !acceptable (file))) |
| 500 |
{ |
| 501 |
/* Either --delete-after was specified, or we loaded this |
| 502 |
(otherwise unneeded because of --spider or rejected by -R) |
| 503 |
HTML file just to harvest its hyperlinks -- in either case, |
| 504 |
delete the local file. */ |
| 505 |
DEBUGP (("Removing file due to %s in recursive_retrieve():\n", |
| 506 |
opt.delete_after ? "--delete-after" : |
| 507 |
(opt.spider ? "--spider" : |
| 508 |
"recursive rejection criteria"))); |
| 509 |
logprintf (LOG_VERBOSE, |
| 510 |
(opt.delete_after || opt.spider |
| 511 |
? _("Removing %s.\n") |
| 512 |
: _("Removing %s since it should be rejected.\n")), |
| 513 |
file); |
| 514 |
if (unlink (file)) |
| 515 |
logprintf (LOG_NOTQUIET, "unlink: %s\n", strerror (errno)); |
| 516 |
logputs (LOG_VERBOSE, "\n"); |
| 517 |
register_delete_file (file); |
| 518 |
} |
| 519 |
|
| 520 |
xfree (url); |
| 521 |
xfree (referer); |
| 522 |
xfree (file); |
| 523 |
iri_free (i); |
| 524 |
} |
| 525 |
|
| 526 |
if (rejectedlog) |
| 527 |
{ |
| 528 |
fclose (rejectedlog); |
| 529 |
rejectedlog = NULL; |
| 530 |
} |
| 531 |
|
| 532 |
/* If anything is left of the queue due to a premature exit, free it |
| 533 |
now. */ |
| 534 |
{ |
| 535 |
char *d1, *d2; |
| 536 |
int d3; |
| 537 |
bool d4, d5; |
| 538 |
struct iri *d6; |
| 539 |
while (url_dequeue (queue, (struct iri **)&d6, |
| 540 |
(const char **)&d1, (const char **)&d2, &d3, &d4, &d5)) |
| 541 |
{ |
| 542 |
iri_free (d6); |
| 543 |
xfree (d1); |
| 544 |
xfree (d2); |
| 545 |
} |
| 546 |
} |
| 547 |
url_queue_delete (queue); |
| 548 |
|
| 549 |
string_set_free (blacklist); |
| 550 |
|
| 551 |
if (opt.quota && total_downloaded_bytes > opt.quota) |
| 552 |
return QUOTEXC; |
| 553 |
else if (status == FWRITEERR) |
| 554 |
return FWRITEERR; |
| 555 |
else |
| 556 |
return RETROK; |
| 557 |
} |
| 558 |
|
| 559 |
/* Based on the context provided by retrieve_tree, decide whether a |
| 560 |
URL is to be descended to. This is only ever called from |
| 561 |
retrieve_tree, but is in a separate function for clarity. |
| 562 |
|
| 563 |
The most expensive checks (such as those for robots) are memoized |
| 564 |
by storing these URLs to BLACKLIST. This may or may not help. It |
| 565 |
will help if those URLs are encountered many times. */ |
| 566 |
|
| 567 |
static reject_reason |
| 568 |
download_child (const struct urlpos *upos, struct url *parent, int depth, |
| 569 |
struct url *start_url_parsed, struct hash_table *blacklist, |
| 570 |
struct iri *iri) |
| 571 |
{ |
| 572 |
struct url *u = upos->url; |
| 573 |
const char *url = u->url; |
| 574 |
bool u_scheme_like_http; |
| 575 |
reject_reason reason = WG_RR_SUCCESS; |
| 576 |
|
| 577 |
DEBUGP (("Deciding whether to enqueue \"%s\".\n", url)); |
| 578 |
|
| 579 |
if (blacklist_contains (blacklist, url)) |
| 580 |
{ |
| 581 |
if (opt.spider) |
| 582 |
{ |
| 583 |
char *referrer = url_string (parent, URL_AUTH_HIDE_PASSWD); |
| 584 |
DEBUGP (("download_child: parent->url is: %s\n", quote (parent->url))); |
| 585 |
visited_url (url, referrer); |
| 586 |
xfree (referrer); |
| 587 |
} |
| 588 |
DEBUGP (("Already on the black list.\n")); |
| 589 |
reason = WG_RR_BLACKLIST; |
| 590 |
goto out; |
| 591 |
} |
| 592 |
|
| 593 |
/* Several things to check for: |
| 594 |
1. if scheme is not https and https_only requested |
| 595 |
2. if scheme is not http, and we don't load it |
| 596 |
3. check for relative links (if relative_only is set) |
| 597 |
4. check for domain |
| 598 |
5. check for no-parent |
| 599 |
6. check for excludes && includes |
| 600 |
7. check for suffix |
| 601 |
8. check for same host (if spanhost is unset), with possible |
| 602 |
gethostbyname baggage |
| 603 |
9. check for robots.txt |
| 604 |
|
| 605 |
Addendum: If the URL is FTP, and it is to be loaded, only the |
| 606 |
domain and suffix settings are "stronger". |
| 607 |
|
| 608 |
Note that .html files will get loaded regardless of suffix rules |
| 609 |
(but that is remedied later with unlink) unless the depth equals |
| 610 |
the maximum depth. |
| 611 |
|
| 612 |
More time- and memory- consuming tests should be put later on |
| 613 |
the list. */ |
| 614 |
|
| 615 |
#ifdef HAVE_SSL |
| 616 |
if (opt.https_only && u->scheme != SCHEME_HTTPS) |
| 617 |
{ |
| 618 |
DEBUGP (("Not following non-HTTPS links.\n")); |
| 619 |
reason = WG_RR_NOTHTTPS; |
| 620 |
goto out; |
| 621 |
} |
| 622 |
#endif |
| 623 |
|
| 624 |
/* Determine whether URL under consideration has a HTTP-like scheme. */ |
| 625 |
u_scheme_like_http = schemes_are_similar_p (u->scheme, SCHEME_HTTP); |
| 626 |
|
| 627 |
/* 1. Schemes other than HTTP are normally not recursed into. */ |
| 628 |
if (!u_scheme_like_http && !((u->scheme == SCHEME_FTP |
| 629 |
#ifdef HAVE_SSL |
| 630 |
|| u->scheme == SCHEME_FTPS |
| 631 |
#endif |
| 632 |
) && opt.follow_ftp)) |
| 633 |
{ |
| 634 |
DEBUGP (("Not following non-HTTP schemes.\n")); |
| 635 |
reason = WG_RR_NONHTTP; |
| 636 |
goto out; |
| 637 |
} |
| 638 |
|
| 639 |
/* 2. If it is an absolute link and they are not followed, throw it |
| 640 |
out. */ |
| 641 |
if (u_scheme_like_http) |
| 642 |
if (opt.relative_only && !upos->link_relative_p) |
| 643 |
{ |
| 644 |
DEBUGP (("It doesn't really look like a relative link.\n")); |
| 645 |
reason = WG_RR_ABSOLUTE; |
| 646 |
goto out; |
| 647 |
} |
| 648 |
|
| 649 |
/* 3. If its domain is not to be accepted/looked-up, chuck it |
| 650 |
out. */ |
| 651 |
if (!accept_domain (u)) |
| 652 |
{ |
| 653 |
DEBUGP (("The domain was not accepted.\n")); |
| 654 |
reason = WG_RR_DOMAIN; |
| 655 |
goto out; |
| 656 |
} |
| 657 |
|
| 658 |
/* 4. Check for parent directory. |
| 659 |
|
| 660 |
If we descended to a different host or changed the scheme, ignore |
| 661 |
opt.no_parent. Also ignore it for documents needed to display |
| 662 |
the parent page when in -p mode. */ |
| 663 |
if (opt.no_parent |
| 664 |
&& schemes_are_similar_p (u->scheme, start_url_parsed->scheme) |
| 665 |
&& 0 == strcasecmp (u->host, start_url_parsed->host) |
| 666 |
&& (u->scheme != start_url_parsed->scheme |
| 667 |
|| u->port == start_url_parsed->port) |
| 668 |
&& !(opt.page_requisites && upos->link_inline_p)) |
| 669 |
{ |
| 670 |
if (!subdir_p (start_url_parsed->dir, u->dir)) |
| 671 |
{ |
| 672 |
DEBUGP (("Going to \"%s\" would escape \"%s\" with no_parent on.\n", |
| 673 |
u->dir, start_url_parsed->dir)); |
| 674 |
reason = WG_RR_PARENT; |
| 675 |
goto out; |
| 676 |
} |
| 677 |
} |
| 678 |
|
| 679 |
/* 5. If the file does not match the acceptance list, or is on the |
| 680 |
rejection list, chuck it out. The same goes for the directory |
| 681 |
exclusion and inclusion lists. */ |
| 682 |
if (opt.includes || opt.excludes) |
| 683 |
{ |
| 684 |
if (!accdir (u->dir)) |
| 685 |
{ |
| 686 |
DEBUGP (("%s (%s) is excluded/not-included.\n", url, u->dir)); |
| 687 |
reason = WG_RR_LIST; |
| 688 |
goto out; |
| 689 |
} |
| 690 |
} |
| 691 |
if (!accept_url (url)) |
| 692 |
{ |
| 693 |
DEBUGP (("%s is excluded/not-included through regex.\n", url)); |
| 694 |
reason = WG_RR_REGEX; |
| 695 |
goto out; |
| 696 |
} |
| 697 |
|
| 698 |
/* 6. Check for acceptance/rejection rules. We ignore these rules |
| 699 |
for directories (no file name to match) and for non-leaf HTMLs, |
| 700 |
which can lead to other files that do need to be downloaded. (-p |
| 701 |
automatically implies non-leaf because with -p we can, if |
| 702 |
necessary, overstep the maximum depth to get the page requisites.) */ |
| 703 |
if (u->file[0] != '\0' |
| 704 |
&& !(has_html_suffix_p (u->file) |
| 705 |
/* The exception only applies to non-leaf HTMLs (but -p |
| 706 |
always implies non-leaf because we can overstep the |
| 707 |
maximum depth to get the requisites): */ |
| 708 |
&& (/* non-leaf */ |
| 709 |
opt.reclevel == INFINITE_RECURSION |
| 710 |
/* also non-leaf */ |
| 711 |
|| depth < opt.reclevel - 1 |
| 712 |
/* -p, which implies non-leaf (see above) */ |
| 713 |
|| opt.page_requisites))) |
| 714 |
{ |
| 715 |
if (!acceptable (u->file)) |
| 716 |
{ |
| 717 |
DEBUGP (("%s (%s) does not match acc/rej rules.\n", |
| 718 |
url, u->file)); |
| 719 |
reason = WG_RR_RULES; |
| 720 |
goto out; |
| 721 |
} |
| 722 |
} |
| 723 |
|
| 724 |
/* 7. */ |
| 725 |
if (schemes_are_similar_p (u->scheme, parent->scheme)) |
| 726 |
if (!opt.spanhost && 0 != strcasecmp (parent->host, u->host)) |
| 727 |
{ |
| 728 |
DEBUGP (("This is not the same hostname as the parent's (%s and %s).\n", |
| 729 |
u->host, parent->host)); |
| 730 |
reason = WG_RR_SPANNEDHOST; |
| 731 |
goto out; |
| 732 |
} |
| 733 |
|
| 734 |
/* 8. */ |
| 735 |
if (opt.use_robots && u_scheme_like_http) |
| 736 |
{ |
| 737 |
struct robot_specs *specs = res_get_specs (u->host, u->port); |
| 738 |
if (!specs) |
| 739 |
{ |
| 740 |
char *rfile; |
| 741 |
if (res_retrieve_file (url, &rfile, iri)) |
| 742 |
{ |
| 743 |
specs = res_parse_from_file (rfile); |
| 744 |
|
| 745 |
/* Delete the robots.txt file if we chose to either delete the |
| 746 |
files after downloading or we're just running a spider or |
| 747 |
we use page requisites or pattern matching. */ |
| 748 |
if (opt.delete_after || opt.spider || match_tail(rfile, ".tmp", false)) |
| 749 |
{ |
| 750 |
logprintf (LOG_VERBOSE, _("Removing %s.\n"), rfile); |
| 751 |
if (unlink (rfile)) |
| 752 |
logprintf (LOG_NOTQUIET, "unlink: %s\n", |
| 753 |
strerror (errno)); |
| 754 |
} |
| 755 |
|
| 756 |
xfree (rfile); |
| 757 |
} |
| 758 |
else |
| 759 |
{ |
| 760 |
/* If we cannot get real specs, at least produce |
| 761 |
dummy ones so that we can register them and stop |
| 762 |
trying to retrieve them. */ |
| 763 |
specs = res_parse ("", 0); |
| 764 |
} |
| 765 |
res_register_specs (u->host, u->port, specs); |
| 766 |
} |
| 767 |
|
| 768 |
/* Now that we have (or don't have) robots.txt specs, we can |
| 769 |
check what they say. */ |
| 770 |
if (!res_match_path (specs, u->path)) |
| 771 |
{ |
| 772 |
DEBUGP (("Not following %s because robots.txt forbids it.\n", url)); |
| 773 |
blacklist_add (blacklist, url); |
| 774 |
reason = WG_RR_ROBOTS; |
| 775 |
goto out; |
| 776 |
} |
| 777 |
} |
| 778 |
|
| 779 |
out: |
| 780 |
|
| 781 |
if (reason == WG_RR_SUCCESS) |
| 782 |
/* The URL has passed all the tests. It can be placed in the |
| 783 |
download queue. */ |
| 784 |
DEBUGP (("Decided to load it.\n")); |
| 785 |
else |
| 786 |
DEBUGP (("Decided NOT to load it.\n")); |
| 787 |
|
| 788 |
return reason; |
| 789 |
} |
| 790 |
|
| 791 |
/* This function determines whether we will consider downloading the |
| 792 |
children of a URL whose download resulted in a redirection, |
| 793 |
possibly to another host, etc. It is needed very rarely, and thus |
| 794 |
it is merely a simple-minded wrapper around download_child. */ |
| 795 |
|
| 796 |
static reject_reason |
| 797 |
descend_redirect (const char *redirected, struct url *orig_parsed, int depth, |
| 798 |
struct url *start_url_parsed, struct hash_table *blacklist, |
| 799 |
struct iri *iri) |
| 800 |
{ |
| 801 |
struct url *new_parsed; |
| 802 |
struct urlpos *upos; |
| 803 |
reject_reason reason; |
| 804 |
|
| 805 |
assert (orig_parsed != NULL); |
| 806 |
|
| 807 |
new_parsed = url_parse (redirected, NULL, NULL, false); |
| 808 |
assert (new_parsed != NULL); |
| 809 |
|
| 810 |
upos = xnew0 (struct urlpos); |
| 811 |
upos->url = new_parsed; |
| 812 |
|
| 813 |
reason = download_child (upos, orig_parsed, depth, |
| 814 |
start_url_parsed, blacklist, iri); |
| 815 |
|
| 816 |
if (reason == WG_RR_SUCCESS) |
| 817 |
blacklist_add (blacklist, upos->url->url); |
| 818 |
else if (reason == WG_RR_LIST || reason == WG_RR_REGEX) |
| 819 |
{ |
| 820 |
DEBUGP (("Ignoring decision for redirects, decided to load it.\n")); |
| 821 |
blacklist_add (blacklist, upos->url->url); |
| 822 |
reason = WG_RR_SUCCESS; |
| 823 |
} |
| 824 |
else |
| 825 |
DEBUGP (("Redirection \"%s\" failed the test.\n", redirected)); |
| 826 |
|
| 827 |
url_free (new_parsed); |
| 828 |
xfree (upos); |
| 829 |
|
| 830 |
return reason; |
| 831 |
} |
| 832 |
|
| 833 |
|
| 834 |
/* This function writes the rejected log header. */ |
| 835 |
static void |
| 836 |
write_reject_log_header (FILE *f) |
| 837 |
{ |
| 838 |
if (!f) |
| 839 |
return; |
| 840 |
|
| 841 |
/* Note: Update this header when columns change in any way. */ |
| 842 |
fprintf (f, "REASON\t" |
| 843 |
"U_URL\tU_SCHEME\tU_HOST\tU_PORT\tU_PATH\tU_PARAMS\tU_QUERY\tU_FRAGMENT\t" |
| 844 |
"P_URL\tP_SCHEME\tP_HOST\tP_PORT\tP_PATH\tP_PARAMS\tP_QUERY\tP_FRAGMENT\n"); |
| 845 |
} |
| 846 |
|
| 847 |
/* This function writes a URL to the reject log. Internal use only. */ |
| 848 |
static void |
| 849 |
write_reject_log_url (FILE *fp, const struct url *url) |
| 850 |
{ |
| 851 |
const char *escaped_str; |
| 852 |
const char *scheme_str; |
| 853 |
|
| 854 |
if (!fp) |
| 855 |
return; |
| 856 |
|
| 857 |
escaped_str = url_escape (url->url); |
| 858 |
|
| 859 |
switch (url->scheme) |
| 860 |
{ |
| 861 |
case SCHEME_HTTP: scheme_str = "SCHEME_HTTP"; break; |
| 862 |
#ifdef HAVE_SSL |
| 863 |
case SCHEME_HTTPS: scheme_str = "SCHEME_HTTPS"; break; |
| 864 |
case SCHEME_FTPS: scheme_str = "SCHEME_FTPS"; break; |
| 865 |
#endif |
| 866 |
case SCHEME_FTP: scheme_str = "SCHEME_FTP"; break; |
| 867 |
default: scheme_str = "SCHEME_INVALID"; break; |
| 868 |
} |
| 869 |
|
| 870 |
fprintf (fp, "%s\t%s\t%s\t%i\t%s\t%s\t%s\t%s", |
| 871 |
escaped_str, |
| 872 |
scheme_str, |
| 873 |
url->host, |
| 874 |
url->port, |
| 875 |
url->path, |
| 876 |
url->params ? url->params : "", |
| 877 |
url->query ? url->query : "", |
| 878 |
url->fragment ? url->fragment : ""); |
| 879 |
|
| 880 |
xfree (escaped_str); |
| 881 |
} |
| 882 |
|
| 883 |
/* This function writes out information on why a URL was rejected and its |
| 884 |
context from download_child such as the URL being rejected and it's |
| 885 |
parent's URL. The format it uses is comma separated values but with tabs. */ |
| 886 |
static void |
| 887 |
write_reject_log_reason (FILE *fp, reject_reason reason, |
| 888 |
const struct url *url, const struct url *parent) |
| 889 |
{ |
| 890 |
const char *reason_str; |
| 891 |
|
| 892 |
if (!fp) |
| 893 |
return; |
| 894 |
|
| 895 |
switch (reason) |
| 896 |
{ |
| 897 |
case WG_RR_SUCCESS: reason_str = "SUCCESS"; break; |
| 898 |
case WG_RR_BLACKLIST: reason_str = "BLACKLIST"; break; |
| 899 |
case WG_RR_NOTHTTPS: reason_str = "NOTHTTPS"; break; |
| 900 |
case WG_RR_NONHTTP: reason_str = "NONHTTP"; break; |
| 901 |
case WG_RR_ABSOLUTE: reason_str = "ABSOLUTE"; break; |
| 902 |
case WG_RR_DOMAIN: reason_str = "DOMAIN"; break; |
| 903 |
case WG_RR_PARENT: reason_str = "PARENT"; break; |
| 904 |
case WG_RR_LIST: reason_str = "LIST"; break; |
| 905 |
case WG_RR_REGEX: reason_str = "REGEX"; break; |
| 906 |
case WG_RR_RULES: reason_str = "RULES"; break; |
| 907 |
case WG_RR_SPANNEDHOST: reason_str = "SPANNEDHOST"; break; |
| 908 |
case WG_RR_ROBOTS: reason_str = "ROBOTS"; break; |
| 909 |
default: reason_str = "UNKNOWN"; break; |
| 910 |
} |
| 911 |
|
| 912 |
fprintf (fp, "%s\t", reason_str); |
| 913 |
write_reject_log_url (fp, url); |
| 914 |
fprintf (fp, "\t"); |
| 915 |
write_reject_log_url (fp, parent); |
| 916 |
fprintf (fp, "\n"); |
| 917 |
} |
| 918 |
|
| 919 |
/* vim:set sts=2 sw=2 cino+={s: */ |