Debian Codegraph @

src/res.c

raw
1 /* Support for Robot Exclusion Standard (RES).
2 Copyright (C) 2001, 2006-2011, 2015, 2018-2024 Free Software
3 Foundation, Inc.
4
5 This file is part of Wget.
6
7 This program is free software; you can redistribute it and/or modify
8 it under the terms of the GNU General Public License as published by
9 the Free Software Foundation; either version 3 of the License, or (at
10 your option) any later version.
11
12 This program is distributed in the hope that it will be useful, but
13 WITHOUT ANY WARRANTY; without even the implied warranty of
14 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 General Public License for more details.
16
17 You should have received a copy of the GNU General Public License
18 along with Wget. If not, see <http://www.gnu.org/licenses/>.
19
20 Additional permission under GNU GPL version 3 section 7
21
22 If you modify this program, or any covered work, by linking or
23 combining it with the OpenSSL project's OpenSSL library (or a
24 modified version of that library), containing parts covered by the
25 terms of the OpenSSL or SSLeay licenses, the Free Software Foundation
26 grants you additional permission to convey the resulting work.
27 Corresponding Source for a non-source form of such a combination
28 shall include the source code for the parts of OpenSSL used as well
29 as that of the covered work. */
30
31 /* This file implements the Robot Exclusion Standard (RES).
32
33 RES is a simple protocol that enables site admins to signalize to
34 the web crawlers that certain parts of the site should not be
35 accessed. All the admin needs to do is create a "robots.txt" file
36 in the web server root, and use simple commands to allow or
37 disallow access to certain parts of the site.
38
39 The first specification was written by Martijn Koster in 1994, and
40 is still available at <http://www.robotstxt.org/orig.html>.
41 In 1996, Martijn wrote an Internet Draft specifying an improved RES
42 specification; however, that work was apparently abandoned since
43 the draft has expired in 1997 and hasn't been replaced since. The
44 draft is available at
45 <http://www.robotstxt.org/norobots-rfc.txt>.
46
47 This file implements RES as specified by the draft. Note that this
48 only handles the "robots.txt" support. The META tag that controls
49 whether the links should be followed is handled in `html-url.c'.
50
51 Known deviations:
52
53 * The end-of-line comment recognition is more in the spirit of the
54 Bourne Shell (as specified by RES-1994). That means that
55 "foo#bar" is taken literally, whereas "foo #bar" is interpreted
56 as "foo". The Draft apparently specifies that both should be
57 interpreted as "foo".
58
59 * We don't recognize sole CR as the line ending.
60
61 * We don't implement expiry mechanism for /robots.txt specs. I
62 consider it non-necessary for a relatively short-lived
63 application such as Wget. Besides, it is highly questionable
64 whether anyone deploys the recommended expiry scheme for
65 robots.txt.
66
67 Entry points are functions res_parse, res_parse_from_file,
68 res_match_path, res_register_specs, res_get_specs, and
69 res_retrieve_file. */
70
71 #include "wget.h"
72
73 #include <stdio.h>
74 #include <stdlib.h>
75 #include <string.h>
76 #include <errno.h>
77 #include <assert.h>
78
79 #include "utils.h"
80 #include "hash.h"
81 #include "url.h"
82 #include "retr.h"
83 #include "res.h"
84 #include "c-strcase.h"
85
86 #ifdef TESTING
87 #include "../tests/unit-tests.h"
88 #endif
89
90 struct path_info {
91 char *path;
92 bool allowedp;
93 bool user_agent_exact_p;
94 };
95
96 struct robot_specs {
97 int count;
98 int size;
99 struct path_info *paths;
100 };
101
102 /* Parsing the robot spec. */
103
104 /* Check whether AGENT (a string of length LENGTH) equals "wget" or
105 "*". If it is either of them, *matches is set to one. If it is
106 "wget", *exact_match is set to one. */
107
108 static void
109 match_user_agent (const char *agent, int length,
110 bool *matches, bool *exact_match)
111 {
112 if (length == 1 && *agent == '*')
113 {
114 *matches = true;
115 *exact_match = false;
116 }
117 else if (BOUNDED_EQUAL_NO_CASE (agent, agent + length, "wget"))
118 {
119 *matches = true;
120 *exact_match = true;
121 }
122 else
123 {
124 *matches = false;
125 *exact_match = false;
126 }
127 }
128
129 /* Add a path specification between PATH_B and PATH_E as one of the
130 paths in SPECS. */
131
132 static void
133 add_path (struct robot_specs *specs, const char *path_b, const char *path_e,
134 bool allowedp, bool exactp)
135 {
136 struct path_info pp;
137 if (path_b < path_e && *path_b == '/')
138 /* Our path representation doesn't use a leading slash, so remove
139 one from theirs. */
140 ++path_b;
141 pp.path = strdupdelim (path_b, path_e);
142 pp.allowedp = allowedp;
143 pp.user_agent_exact_p = exactp;
144 ++specs->count;
145 if (specs->count > specs->size)
146 {
147 if (specs->size == 0)
148 specs->size = 1;
149 else
150 specs->size <<= 1;
151 specs->paths = xrealloc (specs->paths,
152 specs->size * sizeof (struct path_info));
153 }
154 specs->paths[specs->count - 1] = pp;
155 }
156
157 /* Recreate SPECS->paths with only those paths that have
158 user_agent_exact_p set to true. */
159
160 static void
161 prune_non_exact (struct robot_specs *specs)
162 {
163 struct path_info *newpaths;
164 int i, j, cnt;
165 cnt = 0;
166 for (i = 0; i < specs->count; i++)
167 if (specs->paths[i].user_agent_exact_p)
168 ++cnt;
169 newpaths = xnew_array (struct path_info, cnt);
170 for (i = 0, j = 0; i < specs->count; i++)
171 if (specs->paths[i].user_agent_exact_p)
172 newpaths[j++] = specs->paths[i];
173 else
174 xfree (specs->paths[i].path);
175 assert (j == cnt);
176 xfree (specs->paths);
177 specs->paths = newpaths;
178 specs->count = cnt;
179 specs->size = cnt;
180 }
181
182 #define EOL(p) ((p) >= lineend)
183
184 #define SKIP_SPACE(p) do { \
185 while (!EOL (p) && c_isspace (*p)) \
186 ++p; \
187 } while (0)
188
189 #define FIELD_IS(string_literal) \
190 BOUNDED_EQUAL_NO_CASE (field_b, field_e, string_literal)
191
192 /* Parse textual RES specs beginning with SOURCE of length LENGTH.
193 Return a specs objects ready to be fed to res_match_path.
194
195 The parsing itself is trivial, but creating a correct SPECS object
196 is trickier than it seems, because RES is surprisingly byzantine if
197 you attempt to implement it correctly.
198
199 A "record" is a block of one or more `User-Agent' lines followed by
200 one or more `Allow' or `Disallow' lines. Record is accepted by
201 Wget if one of the `User-Agent' lines was "wget", or if the user
202 agent line was "*".
203
204 After all the lines have been read, we examine whether an exact
205 ("wget") user-agent field was specified. If so, we delete all the
206 lines read under "User-Agent: *" blocks because we have our own
207 Wget-specific blocks. This enables the admin to say:
208
209 User-Agent: *
210 Disallow: /
211
212 User-Agent: google
213 User-Agent: wget
214 Disallow: /cgi-bin
215
216 This means that to Wget and to Google, /cgi-bin is disallowed,
217 whereas for all other crawlers, everything is disallowed.
218 res_parse is implemented so that the order of records doesn't
219 matter. In the case above, the "User-Agent: *" could have come
220 after the other one. */
221
222 struct robot_specs *
223 res_parse (const char *source, int length)
224 {
225 int line_count = 1;
226
227 const char *p = source;
228 const char *end = source + length;
229
230 /* true if last applicable user-agent field matches Wget. */
231 bool user_agent_applies = false;
232
233 /* true if last applicable user-agent field *exactly* matches
234 Wget. */
235 bool user_agent_exact = false;
236
237 /* whether we ever encountered exact user agent. */
238 bool found_exact = false;
239
240 /* count of allow/disallow lines in the current "record", i.e. after
241 the last `user-agent' instructions. */
242 int record_count = 0;
243
244 struct robot_specs *specs = xnew0 (struct robot_specs);
245
246 while (1)
247 {
248 const char *lineend, *lineend_real;
249 const char *field_b, *field_e;
250 const char *value_b, *value_e;
251
252 if (p == end)
253 break;
254 lineend_real = memchr (p, '\n', end - p);
255 if (lineend_real)
256 ++lineend_real;
257 else
258 lineend_real = end;
259 lineend = lineend_real;
260
261 /* Before doing anything else, check whether the line is empty
262 or comment-only. */
263 SKIP_SPACE (p);
264 if (EOL (p) || *p == '#')
265 goto next;
266
267 /* Make sure the end-of-line comments are respected by setting
268 lineend to a location preceding the first comment. Real line
269 ending remains in lineend_real. */
270 for (lineend = p; lineend < lineend_real; lineend++)
271 if ((lineend == p || c_isspace (*(lineend - 1)))
272 && *lineend == '#')
273 break;
274
275 /* Ignore trailing whitespace in the same way. */
276 while (lineend > p && c_isspace (*(lineend - 1)))
277 --lineend;
278
279 assert (!EOL (p));
280
281 field_b = p;
282 while (!EOL (p) && (c_isalnum (*p) || *p == '-'))
283 ++p;
284 field_e = p;
285
286 SKIP_SPACE (p);
287 if (field_b == field_e || EOL (p) || *p != ':')
288 {
289 DEBUGP (("Ignoring malformed line %d\n", line_count));
290 goto next;
291 }
292 ++p; /* skip ':' */
293 SKIP_SPACE (p);
294
295 value_b = p;
296 while (!EOL (p))
297 ++p;
298 value_e = p;
299
300 /* Finally, we have a syntactically valid line. */
301 if (FIELD_IS ("user-agent"))
302 {
303 /* We have to support several cases:
304
305 --previous records--
306
307 User-Agent: foo
308 User-Agent: Wget
309 User-Agent: bar
310 ... matching record ...
311
312 User-Agent: baz
313 User-Agent: qux
314 ... non-matching record ...
315
316 User-Agent: *
317 ... matching record, but will be pruned later ...
318
319 We have to respect `User-Agent' at the beginning of each
320 new record simply because we don't know if we're going to
321 encounter "Wget" among the agents or not. Hence,
322 match_user_agent is called when record_count != 0.
323
324 But if record_count is 0, we have to keep calling it
325 until it matches, and if that happens, we must not call
326 it any more, until the next record. Hence the other part
327 of the condition. */
328 if (record_count != 0 || user_agent_applies == false)
329 match_user_agent (value_b, value_e - value_b,
330 &user_agent_applies, &user_agent_exact);
331 if (user_agent_exact)
332 found_exact = true;
333 record_count = 0;
334 }
335 else if (FIELD_IS ("allow"))
336 {
337 if (user_agent_applies)
338 {
339 add_path (specs, value_b, value_e, true, user_agent_exact);
340 }
341 ++record_count;
342 }
343 else if (FIELD_IS ("disallow"))
344 {
345 if (user_agent_applies)
346 {
347 bool allowed = false;
348 if (value_b == value_e)
349 /* Empty "disallow" line means everything is *allowed*! */
350 allowed = true;
351 add_path (specs, value_b, value_e, allowed, user_agent_exact);
352 }
353 ++record_count;
354 }
355 else
356 {
357 DEBUGP (("Ignoring unknown field at line %d\n", line_count));
358 goto next;
359 }
360
361 next:
362 p = lineend_real;
363 ++line_count;
364 }
365
366 if (found_exact)
367 {
368 /* We've encountered an exactly matching user-agent. Throw out
369 all the stuff with user-agent: *. */
370 prune_non_exact (specs);
371 }
372 else if (specs->size > specs->count)
373 {
374 /* add_path normally over-allocates specs->paths. Reallocate it
375 to the correct size in order to conserve some memory. */
376 specs->paths = xrealloc (specs->paths,
377 specs->count * sizeof (struct path_info));
378 specs->size = specs->count;
379 }
380
381 return specs;
382 }
383
384 /* The same like res_parse, but first map the FILENAME into memory,
385 and then parse it. */
386
387 struct robot_specs *
388 res_parse_from_file (const char *filename)
389 {
390 struct robot_specs *specs;
391 struct file_memory *fm = wget_read_file (filename);
392 if (!fm)
393 {
394 logprintf (LOG_NOTQUIET, _("Cannot open %s: %s\n"),
395 filename, strerror (errno));
396 return NULL;
397 }
398 specs = res_parse (fm->content, fm->length);
399 wget_read_file_free (fm);
400 return specs;
401 }
402
403 static void
404 free_specs (struct robot_specs *specs)
405 {
406 int i;
407 for (i = 0; i < specs->count; i++)
408 xfree (specs->paths[i].path);
409 xfree (specs->paths);
410 xfree (specs);
411 }
412
413 /* Matching of a path according to the specs. */
414
415 /* If C is '%' and (ptr[1], ptr[2]) form a hexadecimal number, and if
416 that number is not a numerical representation of '/', decode C and
417 advance the pointer. */
418
419 #define DECODE_MAYBE(c, ptr) do { \
420 if (c == '%' && c_isxdigit (ptr[1]) && c_isxdigit (ptr[2])) \
421 { \
422 unsigned char decoded = X2DIGITS_TO_NUM (ptr[1], ptr[2]); \
423 if (decoded != '/') \
424 { \
425 c = decoded; \
426 ptr += 2; \
427 } \
428 } \
429 } while (0)
430
431 /* The inner matching engine: return true if RECORD_PATH matches
432 URL_PATH. The rules for matching are described at
433 <http://www.robotstxt.org/norobots-rfc.txt>, section 3.2.2. */
434
435 static bool
436 matches (const char *record_path, const char *url_path)
437 {
438 const char *rp = record_path;
439 const char *up = url_path;
440
441 for (; ; ++rp, ++up)
442 {
443 char rc = *rp;
444 char uc = *up;
445 if (!rc)
446 return true;
447 if (!uc)
448 return false;
449 DECODE_MAYBE(rc, rp);
450 DECODE_MAYBE(uc, up);
451 if (rc != uc)
452 return false;
453 }
454 }
455
456 /* Iterate through all paths in SPECS. For the first one that
457 matches, return its allow/reject status. If none matches,
458 retrieval is by default allowed. */
459
460 bool
461 res_match_path (const struct robot_specs *specs, const char *path)
462 {
463 int i;
464 if (!specs)
465 return true;
466 for (i = 0; i < specs->count; i++)
467 if (matches (specs->paths[i].path, path))
468 {
469 bool allowedp = specs->paths[i].allowedp;
470 DEBUGP (("%s path %s because of rule %s.\n",
471 allowedp ? "Allowing" : "Rejecting",
472 path, quote (specs->paths[i].path)));
473 return allowedp;
474 }
475 return true;
476 }
477
478 /* Registering the specs. */
479
480 static struct hash_table *registered_specs;
481
482 /* Register RES specs that below to server on HOST:PORT. They will
483 later be retrievable using res_get_specs. */
484
485 void
486 res_register_specs (const char *host, int port, struct robot_specs *specs)
487 {
488 struct robot_specs *old;
489 char buf[256], *hp, *hp_old;
490
491 if (((unsigned) snprintf (buf, sizeof (buf), "%s:%d", host, port)) >= sizeof (buf))
492 hp = aprintf("%s:%d", host, port);
493 else
494 hp = buf;
495
496 if (!registered_specs)
497 registered_specs = make_nocase_string_hash_table (0);
498
499 if (hash_table_get_pair (registered_specs, hp, &hp_old, &old))
500 {
501 if (hp != buf)
502 xfree (hp);
503 if (old)
504 free_specs (old);
505 hash_table_put (registered_specs, hp_old, specs);
506 }
507 else
508 {
509 hash_table_put (registered_specs, hp == buf ? xstrdup (hp) : hp, specs);
510 }
511 }
512
513 /* Get the specs that belong to HOST:PORT. */
514
515 struct robot_specs *
516 res_get_specs (const char *host, int port)
517 {
518 char buf[256], *hp;
519
520 if (!registered_specs)
521 return NULL;
522
523 if (((unsigned) snprintf (buf, sizeof (buf), "%s:%d", host, port)) >= sizeof (buf))
524 hp = aprintf("%s:%d", host, port);
525 else
526 hp = buf;
527
528 return hash_table_get (registered_specs, hp);
529 }
530
531 /* Loading the robots file. */
532
533 #define RES_SPECS_LOCATION "/robots.txt"
534
535 /* Retrieve the robots.txt from the server root of the server that
536 serves URL. The file will be named according to the currently
537 active rules, and the file name will be returned in *file.
538
539 Return true if robots were retrieved OK, false otherwise. */
540
541 bool
542 res_retrieve_file (const char *url, char **file, struct iri *iri)
543 {
544 struct iri *i = iri_new ();
545 uerr_t err;
546 char *robots_url = uri_merge (url, RES_SPECS_LOCATION);
547 int saved_ts_val = opt.timestamping;
548 int saved_sp_val = opt.spider, url_err;
549 struct url * url_parsed;
550
551 /* Copy server URI encoding for a possible IDNA transformation, no need to
552 encode the full URI in UTF-8 because "robots.txt" is plain ASCII */
553 set_uri_encoding (i, iri->uri_encoding, false);
554 i->utf8_encode = false;
555
556 logputs (LOG_VERBOSE, _("Loading robots.txt; please ignore errors.\n"));
557 *file = NULL;
558 opt.timestamping = false;
559 opt.spider = false;
560
561 url_parsed = url_parse (robots_url, &url_err, i, true);
562 if (!url_parsed)
563 {
564 logprintf (LOG_NOTQUIET, "%s: %s.\n", robots_url, url_error (url_err));
565 err = URLERROR;
566 }
567 else
568 {
569 err = retrieve_url (url_parsed, robots_url, file, NULL, NULL, NULL,
570 false, i, false);
571 url_free(url_parsed);
572 }
573
574 opt.timestamping = saved_ts_val;
575 opt.spider = saved_sp_val;
576 xfree (robots_url);
577 iri_free (i);
578
579 if (err != RETROK && *file != NULL)
580 {
581 /* If the file is not retrieved correctly, but retrieve_url
582 allocated the file name, deallocate is here so that the
583 caller doesn't have to worry about it. */
584 xfree (*file);
585 }
586 return err == RETROK;
587 }
588
589 bool
590 is_robots_txt_url (const char *url)
591 {
592 char *robots_url = uri_merge (url, RES_SPECS_LOCATION);
593 bool ret = are_urls_equal (url, robots_url);
594
595 xfree (robots_url);
596
597 return ret;
598 }
599
600 #if defined DEBUG_MALLOC || defined TESTING
601 void
602 res_cleanup (void)
603 {
604 if (registered_specs)
605 {
606 hash_table_iterator iter;
607 for (hash_table_iterate (registered_specs, &iter);
608 hash_table_iter_next (&iter);
609 )
610 {
611 xfree (iter.key);
612 free_specs (iter.value);
613 }
614 hash_table_destroy (registered_specs);
615 registered_specs = NULL;
616 }
617 }
618 #endif
619
620 #ifdef TESTING
621
622 const char *
623 test_is_robots_txt_url(void)
624 {
625 unsigned i;
626 static const struct {
627 const char *url;
628 bool expected_result;
629 } test_array[] = {
630 { "http://www.yoyodyne.com/robots.txt", true },
631 { "http://www.yoyodyne.com/somepath/", false },
632 { "http://www.yoyodyne.com/somepath/robots.txt", false },
633 };
634
635 for (i = 0; i < countof(test_array); ++i)
636 {
637 mu_assert ("test_is_robots_txt_url: wrong result",
638 is_robots_txt_url (test_array[i].url) == test_array[i].expected_result);
639 }
640
641 return NULL;
642 }
643
644 #endif /* TESTING */
645
646 /*
647 * vim: et ts=2 sw=2
648 */