| 1 |
/* Support for Robot Exclusion Standard (RES). |
| 2 |
Copyright (C) 2001, 2006-2011, 2015, 2018-2024 Free Software |
| 3 |
Foundation, Inc. |
| 4 |
|
| 5 |
This file is part of Wget. |
| 6 |
|
| 7 |
This program is free software; you can redistribute it and/or modify |
| 8 |
it under the terms of the GNU General Public License as published by |
| 9 |
the Free Software Foundation; either version 3 of the License, or (at |
| 10 |
your option) any later version. |
| 11 |
|
| 12 |
This program is distributed in the hope that it will be useful, but |
| 13 |
WITHOUT ANY WARRANTY; without even the implied warranty of |
| 14 |
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU |
| 15 |
General Public License for more details. |
| 16 |
|
| 17 |
You should have received a copy of the GNU General Public License |
| 18 |
along with Wget. If not, see <http://www.gnu.org/licenses/>. |
| 19 |
|
| 20 |
Additional permission under GNU GPL version 3 section 7 |
| 21 |
|
| 22 |
If you modify this program, or any covered work, by linking or |
| 23 |
combining it with the OpenSSL project's OpenSSL library (or a |
| 24 |
modified version of that library), containing parts covered by the |
| 25 |
terms of the OpenSSL or SSLeay licenses, the Free Software Foundation |
| 26 |
grants you additional permission to convey the resulting work. |
| 27 |
Corresponding Source for a non-source form of such a combination |
| 28 |
shall include the source code for the parts of OpenSSL used as well |
| 29 |
as that of the covered work. */ |
| 30 |
|
| 31 |
/* This file implements the Robot Exclusion Standard (RES). |
| 32 |
|
| 33 |
RES is a simple protocol that enables site admins to signalize to |
| 34 |
the web crawlers that certain parts of the site should not be |
| 35 |
accessed. All the admin needs to do is create a "robots.txt" file |
| 36 |
in the web server root, and use simple commands to allow or |
| 37 |
disallow access to certain parts of the site. |
| 38 |
|
| 39 |
The first specification was written by Martijn Koster in 1994, and |
| 40 |
is still available at <http://www.robotstxt.org/orig.html>. |
| 41 |
In 1996, Martijn wrote an Internet Draft specifying an improved RES |
| 42 |
specification; however, that work was apparently abandoned since |
| 43 |
the draft has expired in 1997 and hasn't been replaced since. The |
| 44 |
draft is available at |
| 45 |
<http://www.robotstxt.org/norobots-rfc.txt>. |
| 46 |
|
| 47 |
This file implements RES as specified by the draft. Note that this |
| 48 |
only handles the "robots.txt" support. The META tag that controls |
| 49 |
whether the links should be followed is handled in `html-url.c'. |
| 50 |
|
| 51 |
Known deviations: |
| 52 |
|
| 53 |
* The end-of-line comment recognition is more in the spirit of the |
| 54 |
Bourne Shell (as specified by RES-1994). That means that |
| 55 |
"foo#bar" is taken literally, whereas "foo #bar" is interpreted |
| 56 |
as "foo". The Draft apparently specifies that both should be |
| 57 |
interpreted as "foo". |
| 58 |
|
| 59 |
* We don't recognize sole CR as the line ending. |
| 60 |
|
| 61 |
* We don't implement expiry mechanism for /robots.txt specs. I |
| 62 |
consider it non-necessary for a relatively short-lived |
| 63 |
application such as Wget. Besides, it is highly questionable |
| 64 |
whether anyone deploys the recommended expiry scheme for |
| 65 |
robots.txt. |
| 66 |
|
| 67 |
Entry points are functions res_parse, res_parse_from_file, |
| 68 |
res_match_path, res_register_specs, res_get_specs, and |
| 69 |
res_retrieve_file. */ |
| 70 |
|
| 71 |
#include "wget.h" |
| 72 |
|
| 73 |
#include <stdio.h> |
| 74 |
#include <stdlib.h> |
| 75 |
#include <string.h> |
| 76 |
#include <errno.h> |
| 77 |
#include <assert.h> |
| 78 |
|
| 79 |
#include "utils.h" |
| 80 |
#include "hash.h" |
| 81 |
#include "url.h" |
| 82 |
#include "retr.h" |
| 83 |
#include "res.h" |
| 84 |
#include "c-strcase.h" |
| 85 |
|
| 86 |
#ifdef TESTING |
| 87 |
#include "../tests/unit-tests.h" |
| 88 |
#endif |
| 89 |
|
| 90 |
struct path_info { |
| 91 |
char *path; |
| 92 |
bool allowedp; |
| 93 |
bool user_agent_exact_p; |
| 94 |
}; |
| 95 |
|
| 96 |
struct robot_specs { |
| 97 |
int count; |
| 98 |
int size; |
| 99 |
struct path_info *paths; |
| 100 |
}; |
| 101 |
|
| 102 |
/* Parsing the robot spec. */ |
| 103 |
|
| 104 |
/* Check whether AGENT (a string of length LENGTH) equals "wget" or |
| 105 |
"*". If it is either of them, *matches is set to one. If it is |
| 106 |
"wget", *exact_match is set to one. */ |
| 107 |
|
| 108 |
static void |
| 109 |
match_user_agent (const char *agent, int length, |
| 110 |
bool *matches, bool *exact_match) |
| 111 |
{ |
| 112 |
if (length == 1 && *agent == '*') |
| 113 |
{ |
| 114 |
*matches = true; |
| 115 |
*exact_match = false; |
| 116 |
} |
| 117 |
else if (BOUNDED_EQUAL_NO_CASE (agent, agent + length, "wget")) |
| 118 |
{ |
| 119 |
*matches = true; |
| 120 |
*exact_match = true; |
| 121 |
} |
| 122 |
else |
| 123 |
{ |
| 124 |
*matches = false; |
| 125 |
*exact_match = false; |
| 126 |
} |
| 127 |
} |
| 128 |
|
| 129 |
/* Add a path specification between PATH_B and PATH_E as one of the |
| 130 |
paths in SPECS. */ |
| 131 |
|
| 132 |
static void |
| 133 |
add_path (struct robot_specs *specs, const char *path_b, const char *path_e, |
| 134 |
bool allowedp, bool exactp) |
| 135 |
{ |
| 136 |
struct path_info pp; |
| 137 |
if (path_b < path_e && *path_b == '/') |
| 138 |
/* Our path representation doesn't use a leading slash, so remove |
| 139 |
one from theirs. */ |
| 140 |
++path_b; |
| 141 |
pp.path = strdupdelim (path_b, path_e); |
| 142 |
pp.allowedp = allowedp; |
| 143 |
pp.user_agent_exact_p = exactp; |
| 144 |
++specs->count; |
| 145 |
if (specs->count > specs->size) |
| 146 |
{ |
| 147 |
if (specs->size == 0) |
| 148 |
specs->size = 1; |
| 149 |
else |
| 150 |
specs->size <<= 1; |
| 151 |
specs->paths = xrealloc (specs->paths, |
| 152 |
specs->size * sizeof (struct path_info)); |
| 153 |
} |
| 154 |
specs->paths[specs->count - 1] = pp; |
| 155 |
} |
| 156 |
|
| 157 |
/* Recreate SPECS->paths with only those paths that have |
| 158 |
user_agent_exact_p set to true. */ |
| 159 |
|
| 160 |
static void |
| 161 |
prune_non_exact (struct robot_specs *specs) |
| 162 |
{ |
| 163 |
struct path_info *newpaths; |
| 164 |
int i, j, cnt; |
| 165 |
cnt = 0; |
| 166 |
for (i = 0; i < specs->count; i++) |
| 167 |
if (specs->paths[i].user_agent_exact_p) |
| 168 |
++cnt; |
| 169 |
newpaths = xnew_array (struct path_info, cnt); |
| 170 |
for (i = 0, j = 0; i < specs->count; i++) |
| 171 |
if (specs->paths[i].user_agent_exact_p) |
| 172 |
newpaths[j++] = specs->paths[i]; |
| 173 |
else |
| 174 |
xfree (specs->paths[i].path); |
| 175 |
assert (j == cnt); |
| 176 |
xfree (specs->paths); |
| 177 |
specs->paths = newpaths; |
| 178 |
specs->count = cnt; |
| 179 |
specs->size = cnt; |
| 180 |
} |
| 181 |
|
| 182 |
#define EOL(p) ((p) >= lineend) |
| 183 |
|
| 184 |
#define SKIP_SPACE(p) do { \ |
| 185 |
while (!EOL (p) && c_isspace (*p)) \ |
| 186 |
++p; \ |
| 187 |
} while (0) |
| 188 |
|
| 189 |
#define FIELD_IS(string_literal) \ |
| 190 |
BOUNDED_EQUAL_NO_CASE (field_b, field_e, string_literal) |
| 191 |
|
| 192 |
/* Parse textual RES specs beginning with SOURCE of length LENGTH. |
| 193 |
Return a specs objects ready to be fed to res_match_path. |
| 194 |
|
| 195 |
The parsing itself is trivial, but creating a correct SPECS object |
| 196 |
is trickier than it seems, because RES is surprisingly byzantine if |
| 197 |
you attempt to implement it correctly. |
| 198 |
|
| 199 |
A "record" is a block of one or more `User-Agent' lines followed by |
| 200 |
one or more `Allow' or `Disallow' lines. Record is accepted by |
| 201 |
Wget if one of the `User-Agent' lines was "wget", or if the user |
| 202 |
agent line was "*". |
| 203 |
|
| 204 |
After all the lines have been read, we examine whether an exact |
| 205 |
("wget") user-agent field was specified. If so, we delete all the |
| 206 |
lines read under "User-Agent: *" blocks because we have our own |
| 207 |
Wget-specific blocks. This enables the admin to say: |
| 208 |
|
| 209 |
User-Agent: * |
| 210 |
Disallow: / |
| 211 |
|
| 212 |
User-Agent: google |
| 213 |
User-Agent: wget |
| 214 |
Disallow: /cgi-bin |
| 215 |
|
| 216 |
This means that to Wget and to Google, /cgi-bin is disallowed, |
| 217 |
whereas for all other crawlers, everything is disallowed. |
| 218 |
res_parse is implemented so that the order of records doesn't |
| 219 |
matter. In the case above, the "User-Agent: *" could have come |
| 220 |
after the other one. */ |
| 221 |
|
| 222 |
struct robot_specs * |
| 223 |
res_parse (const char *source, int length) |
| 224 |
{ |
| 225 |
int line_count = 1; |
| 226 |
|
| 227 |
const char *p = source; |
| 228 |
const char *end = source + length; |
| 229 |
|
| 230 |
/* true if last applicable user-agent field matches Wget. */ |
| 231 |
bool user_agent_applies = false; |
| 232 |
|
| 233 |
/* true if last applicable user-agent field *exactly* matches |
| 234 |
Wget. */ |
| 235 |
bool user_agent_exact = false; |
| 236 |
|
| 237 |
/* whether we ever encountered exact user agent. */ |
| 238 |
bool found_exact = false; |
| 239 |
|
| 240 |
/* count of allow/disallow lines in the current "record", i.e. after |
| 241 |
the last `user-agent' instructions. */ |
| 242 |
int record_count = 0; |
| 243 |
|
| 244 |
struct robot_specs *specs = xnew0 (struct robot_specs); |
| 245 |
|
| 246 |
while (1) |
| 247 |
{ |
| 248 |
const char *lineend, *lineend_real; |
| 249 |
const char *field_b, *field_e; |
| 250 |
const char *value_b, *value_e; |
| 251 |
|
| 252 |
if (p == end) |
| 253 |
break; |
| 254 |
lineend_real = memchr (p, '\n', end - p); |
| 255 |
if (lineend_real) |
| 256 |
++lineend_real; |
| 257 |
else |
| 258 |
lineend_real = end; |
| 259 |
lineend = lineend_real; |
| 260 |
|
| 261 |
/* Before doing anything else, check whether the line is empty |
| 262 |
or comment-only. */ |
| 263 |
SKIP_SPACE (p); |
| 264 |
if (EOL (p) || *p == '#') |
| 265 |
goto next; |
| 266 |
|
| 267 |
/* Make sure the end-of-line comments are respected by setting |
| 268 |
lineend to a location preceding the first comment. Real line |
| 269 |
ending remains in lineend_real. */ |
| 270 |
for (lineend = p; lineend < lineend_real; lineend++) |
| 271 |
if ((lineend == p || c_isspace (*(lineend - 1))) |
| 272 |
&& *lineend == '#') |
| 273 |
break; |
| 274 |
|
| 275 |
/* Ignore trailing whitespace in the same way. */ |
| 276 |
while (lineend > p && c_isspace (*(lineend - 1))) |
| 277 |
--lineend; |
| 278 |
|
| 279 |
assert (!EOL (p)); |
| 280 |
|
| 281 |
field_b = p; |
| 282 |
while (!EOL (p) && (c_isalnum (*p) || *p == '-')) |
| 283 |
++p; |
| 284 |
field_e = p; |
| 285 |
|
| 286 |
SKIP_SPACE (p); |
| 287 |
if (field_b == field_e || EOL (p) || *p != ':') |
| 288 |
{ |
| 289 |
DEBUGP (("Ignoring malformed line %d\n", line_count)); |
| 290 |
goto next; |
| 291 |
} |
| 292 |
++p; /* skip ':' */ |
| 293 |
SKIP_SPACE (p); |
| 294 |
|
| 295 |
value_b = p; |
| 296 |
while (!EOL (p)) |
| 297 |
++p; |
| 298 |
value_e = p; |
| 299 |
|
| 300 |
/* Finally, we have a syntactically valid line. */ |
| 301 |
if (FIELD_IS ("user-agent")) |
| 302 |
{ |
| 303 |
/* We have to support several cases: |
| 304 |
|
| 305 |
--previous records-- |
| 306 |
|
| 307 |
User-Agent: foo |
| 308 |
User-Agent: Wget |
| 309 |
User-Agent: bar |
| 310 |
... matching record ... |
| 311 |
|
| 312 |
User-Agent: baz |
| 313 |
User-Agent: qux |
| 314 |
... non-matching record ... |
| 315 |
|
| 316 |
User-Agent: * |
| 317 |
... matching record, but will be pruned later ... |
| 318 |
|
| 319 |
We have to respect `User-Agent' at the beginning of each |
| 320 |
new record simply because we don't know if we're going to |
| 321 |
encounter "Wget" among the agents or not. Hence, |
| 322 |
match_user_agent is called when record_count != 0. |
| 323 |
|
| 324 |
But if record_count is 0, we have to keep calling it |
| 325 |
until it matches, and if that happens, we must not call |
| 326 |
it any more, until the next record. Hence the other part |
| 327 |
of the condition. */ |
| 328 |
if (record_count != 0 || user_agent_applies == false) |
| 329 |
match_user_agent (value_b, value_e - value_b, |
| 330 |
&user_agent_applies, &user_agent_exact); |
| 331 |
if (user_agent_exact) |
| 332 |
found_exact = true; |
| 333 |
record_count = 0; |
| 334 |
} |
| 335 |
else if (FIELD_IS ("allow")) |
| 336 |
{ |
| 337 |
if (user_agent_applies) |
| 338 |
{ |
| 339 |
add_path (specs, value_b, value_e, true, user_agent_exact); |
| 340 |
} |
| 341 |
++record_count; |
| 342 |
} |
| 343 |
else if (FIELD_IS ("disallow")) |
| 344 |
{ |
| 345 |
if (user_agent_applies) |
| 346 |
{ |
| 347 |
bool allowed = false; |
| 348 |
if (value_b == value_e) |
| 349 |
/* Empty "disallow" line means everything is *allowed*! */ |
| 350 |
allowed = true; |
| 351 |
add_path (specs, value_b, value_e, allowed, user_agent_exact); |
| 352 |
} |
| 353 |
++record_count; |
| 354 |
} |
| 355 |
else |
| 356 |
{ |
| 357 |
DEBUGP (("Ignoring unknown field at line %d\n", line_count)); |
| 358 |
goto next; |
| 359 |
} |
| 360 |
|
| 361 |
next: |
| 362 |
p = lineend_real; |
| 363 |
++line_count; |
| 364 |
} |
| 365 |
|
| 366 |
if (found_exact) |
| 367 |
{ |
| 368 |
/* We've encountered an exactly matching user-agent. Throw out |
| 369 |
all the stuff with user-agent: *. */ |
| 370 |
prune_non_exact (specs); |
| 371 |
} |
| 372 |
else if (specs->size > specs->count) |
| 373 |
{ |
| 374 |
/* add_path normally over-allocates specs->paths. Reallocate it |
| 375 |
to the correct size in order to conserve some memory. */ |
| 376 |
specs->paths = xrealloc (specs->paths, |
| 377 |
specs->count * sizeof (struct path_info)); |
| 378 |
specs->size = specs->count; |
| 379 |
} |
| 380 |
|
| 381 |
return specs; |
| 382 |
} |
| 383 |
|
| 384 |
/* The same like res_parse, but first map the FILENAME into memory, |
| 385 |
and then parse it. */ |
| 386 |
|
| 387 |
struct robot_specs * |
| 388 |
res_parse_from_file (const char *filename) |
| 389 |
{ |
| 390 |
struct robot_specs *specs; |
| 391 |
struct file_memory *fm = wget_read_file (filename); |
| 392 |
if (!fm) |
| 393 |
{ |
| 394 |
logprintf (LOG_NOTQUIET, _("Cannot open %s: %s\n"), |
| 395 |
filename, strerror (errno)); |
| 396 |
return NULL; |
| 397 |
} |
| 398 |
specs = res_parse (fm->content, fm->length); |
| 399 |
wget_read_file_free (fm); |
| 400 |
return specs; |
| 401 |
} |
| 402 |
|
| 403 |
static void |
| 404 |
free_specs (struct robot_specs *specs) |
| 405 |
{ |
| 406 |
int i; |
| 407 |
for (i = 0; i < specs->count; i++) |
| 408 |
xfree (specs->paths[i].path); |
| 409 |
xfree (specs->paths); |
| 410 |
xfree (specs); |
| 411 |
} |
| 412 |
|
| 413 |
/* Matching of a path according to the specs. */ |
| 414 |
|
| 415 |
/* If C is '%' and (ptr[1], ptr[2]) form a hexadecimal number, and if |
| 416 |
that number is not a numerical representation of '/', decode C and |
| 417 |
advance the pointer. */ |
| 418 |
|
| 419 |
#define DECODE_MAYBE(c, ptr) do { \ |
| 420 |
if (c == '%' && c_isxdigit (ptr[1]) && c_isxdigit (ptr[2])) \ |
| 421 |
{ \ |
| 422 |
unsigned char decoded = X2DIGITS_TO_NUM (ptr[1], ptr[2]); \ |
| 423 |
if (decoded != '/') \ |
| 424 |
{ \ |
| 425 |
c = decoded; \ |
| 426 |
ptr += 2; \ |
| 427 |
} \ |
| 428 |
} \ |
| 429 |
} while (0) |
| 430 |
|
| 431 |
/* The inner matching engine: return true if RECORD_PATH matches |
| 432 |
URL_PATH. The rules for matching are described at |
| 433 |
<http://www.robotstxt.org/norobots-rfc.txt>, section 3.2.2. */ |
| 434 |
|
| 435 |
static bool |
| 436 |
matches (const char *record_path, const char *url_path) |
| 437 |
{ |
| 438 |
const char *rp = record_path; |
| 439 |
const char *up = url_path; |
| 440 |
|
| 441 |
for (; ; ++rp, ++up) |
| 442 |
{ |
| 443 |
char rc = *rp; |
| 444 |
char uc = *up; |
| 445 |
if (!rc) |
| 446 |
return true; |
| 447 |
if (!uc) |
| 448 |
return false; |
| 449 |
DECODE_MAYBE(rc, rp); |
| 450 |
DECODE_MAYBE(uc, up); |
| 451 |
if (rc != uc) |
| 452 |
return false; |
| 453 |
} |
| 454 |
} |
| 455 |
|
| 456 |
/* Iterate through all paths in SPECS. For the first one that |
| 457 |
matches, return its allow/reject status. If none matches, |
| 458 |
retrieval is by default allowed. */ |
| 459 |
|
| 460 |
bool |
| 461 |
res_match_path (const struct robot_specs *specs, const char *path) |
| 462 |
{ |
| 463 |
int i; |
| 464 |
if (!specs) |
| 465 |
return true; |
| 466 |
for (i = 0; i < specs->count; i++) |
| 467 |
if (matches (specs->paths[i].path, path)) |
| 468 |
{ |
| 469 |
bool allowedp = specs->paths[i].allowedp; |
| 470 |
DEBUGP (("%s path %s because of rule %s.\n", |
| 471 |
allowedp ? "Allowing" : "Rejecting", |
| 472 |
path, quote (specs->paths[i].path))); |
| 473 |
return allowedp; |
| 474 |
} |
| 475 |
return true; |
| 476 |
} |
| 477 |
|
| 478 |
/* Registering the specs. */ |
| 479 |
|
| 480 |
static struct hash_table *registered_specs; |
| 481 |
|
| 482 |
/* Register RES specs that below to server on HOST:PORT. They will |
| 483 |
later be retrievable using res_get_specs. */ |
| 484 |
|
| 485 |
void |
| 486 |
res_register_specs (const char *host, int port, struct robot_specs *specs) |
| 487 |
{ |
| 488 |
struct robot_specs *old; |
| 489 |
char buf[256], *hp, *hp_old; |
| 490 |
|
| 491 |
if (((unsigned) snprintf (buf, sizeof (buf), "%s:%d", host, port)) >= sizeof (buf)) |
| 492 |
hp = aprintf("%s:%d", host, port); |
| 493 |
else |
| 494 |
hp = buf; |
| 495 |
|
| 496 |
if (!registered_specs) |
| 497 |
registered_specs = make_nocase_string_hash_table (0); |
| 498 |
|
| 499 |
if (hash_table_get_pair (registered_specs, hp, &hp_old, &old)) |
| 500 |
{ |
| 501 |
if (hp != buf) |
| 502 |
xfree (hp); |
| 503 |
if (old) |
| 504 |
free_specs (old); |
| 505 |
hash_table_put (registered_specs, hp_old, specs); |
| 506 |
} |
| 507 |
else |
| 508 |
{ |
| 509 |
hash_table_put (registered_specs, hp == buf ? xstrdup (hp) : hp, specs); |
| 510 |
} |
| 511 |
} |
| 512 |
|
| 513 |
/* Get the specs that belong to HOST:PORT. */ |
| 514 |
|
| 515 |
struct robot_specs * |
| 516 |
res_get_specs (const char *host, int port) |
| 517 |
{ |
| 518 |
char buf[256], *hp; |
| 519 |
|
| 520 |
if (!registered_specs) |
| 521 |
return NULL; |
| 522 |
|
| 523 |
if (((unsigned) snprintf (buf, sizeof (buf), "%s:%d", host, port)) >= sizeof (buf)) |
| 524 |
hp = aprintf("%s:%d", host, port); |
| 525 |
else |
| 526 |
hp = buf; |
| 527 |
|
| 528 |
return hash_table_get (registered_specs, hp); |
| 529 |
} |
| 530 |
|
| 531 |
/* Loading the robots file. */ |
| 532 |
|
| 533 |
#define RES_SPECS_LOCATION "/robots.txt" |
| 534 |
|
| 535 |
/* Retrieve the robots.txt from the server root of the server that |
| 536 |
serves URL. The file will be named according to the currently |
| 537 |
active rules, and the file name will be returned in *file. |
| 538 |
|
| 539 |
Return true if robots were retrieved OK, false otherwise. */ |
| 540 |
|
| 541 |
bool |
| 542 |
res_retrieve_file (const char *url, char **file, struct iri *iri) |
| 543 |
{ |
| 544 |
struct iri *i = iri_new (); |
| 545 |
uerr_t err; |
| 546 |
char *robots_url = uri_merge (url, RES_SPECS_LOCATION); |
| 547 |
int saved_ts_val = opt.timestamping; |
| 548 |
int saved_sp_val = opt.spider, url_err; |
| 549 |
struct url * url_parsed; |
| 550 |
|
| 551 |
/* Copy server URI encoding for a possible IDNA transformation, no need to |
| 552 |
encode the full URI in UTF-8 because "robots.txt" is plain ASCII */ |
| 553 |
set_uri_encoding (i, iri->uri_encoding, false); |
| 554 |
i->utf8_encode = false; |
| 555 |
|
| 556 |
logputs (LOG_VERBOSE, _("Loading robots.txt; please ignore errors.\n")); |
| 557 |
*file = NULL; |
| 558 |
opt.timestamping = false; |
| 559 |
opt.spider = false; |
| 560 |
|
| 561 |
url_parsed = url_parse (robots_url, &url_err, i, true); |
| 562 |
if (!url_parsed) |
| 563 |
{ |
| 564 |
logprintf (LOG_NOTQUIET, "%s: %s.\n", robots_url, url_error (url_err)); |
| 565 |
err = URLERROR; |
| 566 |
} |
| 567 |
else |
| 568 |
{ |
| 569 |
err = retrieve_url (url_parsed, robots_url, file, NULL, NULL, NULL, |
| 570 |
false, i, false); |
| 571 |
url_free(url_parsed); |
| 572 |
} |
| 573 |
|
| 574 |
opt.timestamping = saved_ts_val; |
| 575 |
opt.spider = saved_sp_val; |
| 576 |
xfree (robots_url); |
| 577 |
iri_free (i); |
| 578 |
|
| 579 |
if (err != RETROK && *file != NULL) |
| 580 |
{ |
| 581 |
/* If the file is not retrieved correctly, but retrieve_url |
| 582 |
allocated the file name, deallocate is here so that the |
| 583 |
caller doesn't have to worry about it. */ |
| 584 |
xfree (*file); |
| 585 |
} |
| 586 |
return err == RETROK; |
| 587 |
} |
| 588 |
|
| 589 |
bool |
| 590 |
is_robots_txt_url (const char *url) |
| 591 |
{ |
| 592 |
char *robots_url = uri_merge (url, RES_SPECS_LOCATION); |
| 593 |
bool ret = are_urls_equal (url, robots_url); |
| 594 |
|
| 595 |
xfree (robots_url); |
| 596 |
|
| 597 |
return ret; |
| 598 |
} |
| 599 |
|
| 600 |
#if defined DEBUG_MALLOC || defined TESTING |
| 601 |
void |
| 602 |
res_cleanup (void) |
| 603 |
{ |
| 604 |
if (registered_specs) |
| 605 |
{ |
| 606 |
hash_table_iterator iter; |
| 607 |
for (hash_table_iterate (registered_specs, &iter); |
| 608 |
hash_table_iter_next (&iter); |
| 609 |
) |
| 610 |
{ |
| 611 |
xfree (iter.key); |
| 612 |
free_specs (iter.value); |
| 613 |
} |
| 614 |
hash_table_destroy (registered_specs); |
| 615 |
registered_specs = NULL; |
| 616 |
} |
| 617 |
} |
| 618 |
#endif |
| 619 |
|
| 620 |
#ifdef TESTING |
| 621 |
|
| 622 |
const char * |
| 623 |
test_is_robots_txt_url(void) |
| 624 |
{ |
| 625 |
unsigned i; |
| 626 |
static const struct { |
| 627 |
const char *url; |
| 628 |
bool expected_result; |
| 629 |
} test_array[] = { |
| 630 |
{ "http://www.yoyodyne.com/robots.txt", true }, |
| 631 |
{ "http://www.yoyodyne.com/somepath/", false }, |
| 632 |
{ "http://www.yoyodyne.com/somepath/robots.txt", false }, |
| 633 |
}; |
| 634 |
|
| 635 |
for (i = 0; i < countof(test_array); ++i) |
| 636 |
{ |
| 637 |
mu_assert ("test_is_robots_txt_url: wrong result", |
| 638 |
is_robots_txt_url (test_array[i].url) == test_array[i].expected_result); |
| 639 |
} |
| 640 |
|
| 641 |
return NULL; |
| 642 |
} |
| 643 |
|
| 644 |
#endif /* TESTING */ |
| 645 |
|
| 646 |
/* |
| 647 |
* vim: et ts=2 sw=2 |
| 648 |
*/ |