Debian Codegraph @

src/url.c

raw
1 /* URL handling.
2 Copyright (C) 1996-2011, 2015, 2018-2024 Free Software Foundation,
3 Inc.
4
5 This file is part of GNU Wget.
6
7 GNU Wget is free software; you can redistribute it and/or modify
8 it under the terms of the GNU General Public License as published by
9 the Free Software Foundation; either version 3 of the License, or (at
10 your option) any later version.
11
12 GNU Wget is distributed in the hope that it will be useful,
13 but WITHOUT ANY WARRANTY; without even the implied warranty of
14 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 GNU General Public License for more details.
16
17 You should have received a copy of the GNU General Public License
18 along with Wget. If not, see <http://www.gnu.org/licenses/>.
19
20 Additional permission under GNU GPL version 3 section 7
21
22 If you modify this program, or any covered work, by linking or
23 combining it with the OpenSSL project's OpenSSL library (or a
24 modified version of that library), containing parts covered by the
25 terms of the OpenSSL or SSLeay licenses, the Free Software Foundation
26 grants you additional permission to convey the resulting work.
27 Corresponding Source for a non-source form of such a combination
28 shall include the source code for the parts of OpenSSL used as well
29 as that of the covered work. */
30
31 #include "wget.h"
32
33 #include <stdio.h>
34 #include <stdlib.h>
35 #include <string.h>
36 #include <unistd.h>
37 #include <errno.h>
38 #include <assert.h>
39
40 #include "utils.h"
41 #include "url.h"
42 #include "host.h" /* for is_valid_ipv6_address */
43 #include "c-strcase.h"
44 #include "c-ctype.h"
45
46 #ifdef HAVE_ICONV
47 # include <iconv.h>
48 #endif
49 #include <langinfo.h>
50
51 #ifdef __VMS
52 #include "vms.h"
53 #endif /* def __VMS */
54
55 #ifdef TESTING
56 #include "../tests/unit-tests.h"
57 #endif
58
59 enum {
60 scm_disabled = 1, /* for https when OpenSSL fails to init. */
61 scm_has_params = 2, /* whether scheme has ;params */
62 scm_has_query = 4, /* whether scheme has ?query */
63 scm_has_fragment = 8 /* whether scheme has #fragment */
64 };
65
66 struct scheme_data
67 {
68 /* Short name of the scheme, such as "http" or "ftp". */
69 const char *name;
70 /* Leading string that identifies the scheme, such as "https://". */
71 const char *leading_string;
72 /* Default port of the scheme when none is specified. */
73 int default_port;
74 /* Various flags. */
75 int flags;
76 };
77
78 /* Supported schemes: */
79 static struct scheme_data supported_schemes[] =
80 {
81 { "http", "http://", DEFAULT_HTTP_PORT, scm_has_query|scm_has_fragment },
82 #ifdef HAVE_SSL
83 { "https", "https://", DEFAULT_HTTPS_PORT, scm_has_query|scm_has_fragment },
84 #endif
85 { "ftp", "ftp://", DEFAULT_FTP_PORT, scm_has_params|scm_has_fragment },
86 #ifdef HAVE_SSL
87 /*
88 * Explicit FTPS uses the same port as FTP.
89 * Implicit FTPS has its own port (990), but it is disabled by default.
90 */
91 { "ftps", "ftps://", DEFAULT_FTP_PORT, scm_has_params|scm_has_fragment },
92 #endif
93
94 /* SCHEME_INVALID */
95 { NULL, NULL, -1, 0 }
96 };
97
98 /* Forward declarations: */
99
100 static bool path_simplify (enum url_scheme, char *);
101
102 /* Support for escaping and unescaping of URL strings. */
103
104 /* Table of "reserved" and "unsafe" characters. Those terms are
105 rfc1738-speak, as such largely obsoleted by rfc2396 and later
106 specs, but the general idea remains.
107
108 A reserved character is the one that you can't decode without
109 changing the meaning of the URL. For example, you can't decode
110 "/foo/%2f/bar" into "/foo///bar" because the number and contents of
111 path components is different. Non-reserved characters can be
112 changed, so "/foo/%78/bar" is safe to change to "/foo/x/bar". The
113 unsafe characters are loosely based on rfc1738, plus "$" and ",",
114 as recommended by rfc2396, and minus "~", which is very frequently
115 used (and sometimes unrecognized as %7E by broken servers).
116
117 An unsafe character is the one that should be encoded when URLs are
118 placed in foreign environments. E.g. space and newline are unsafe
119 in HTTP contexts because HTTP uses them as separator and line
120 terminator, so they must be encoded to %20 and %0A respectively.
121 "*" is unsafe in shell context, etc.
122
123 We determine whether a character is unsafe through static table
124 lookup. This code assumes ASCII character set and 8-bit chars. */
125
126 enum {
127 /* rfc1738 reserved chars + "$" and ",". */
128 urlchr_reserved = 1,
129
130 /* rfc1738 unsafe chars, plus non-printables. */
131 urlchr_unsafe = 2
132 };
133
134 #define urlchr_test(c, mask) (urlchr_table[(unsigned char)(c)] & (mask))
135 #define URL_RESERVED_CHAR(c) urlchr_test(c, urlchr_reserved)
136 #define URL_UNSAFE_CHAR(c) urlchr_test(c, urlchr_unsafe)
137
138 /* Shorthands for the table: */
139 #define R urlchr_reserved
140 #define U urlchr_unsafe
141 #define RU R|U
142
143 static const unsigned char urlchr_table[256] =
144 {
145 U, U, U, U, U, U, U, U, /* NUL SOH STX ETX EOT ENQ ACK BEL */
146 U, U, U, U, U, U, U, U, /* BS HT LF VT FF CR SO SI */
147 U, U, U, U, U, U, U, U, /* DLE DC1 DC2 DC3 DC4 NAK SYN ETB */
148 U, U, U, U, U, U, U, U, /* CAN EM SUB ESC FS GS RS US */
149 U, 0, U, RU, R, U, R, 0, /* SP ! " # $ % & ' */
150 0, 0, 0, R, R, 0, 0, R, /* ( ) * + , - . / */
151 0, 0, 0, 0, 0, 0, 0, 0, /* 0 1 2 3 4 5 6 7 */
152 0, 0, RU, R, U, R, U, R, /* 8 9 : ; < = > ? */
153 RU, 0, 0, 0, 0, 0, 0, 0, /* @ A B C D E F G */
154 0, 0, 0, 0, 0, 0, 0, 0, /* H I J K L M N O */
155 0, 0, 0, 0, 0, 0, 0, 0, /* P Q R S T U V W */
156 0, 0, 0, RU, U, RU, U, 0, /* X Y Z [ \ ] ^ _ */
157 U, 0, 0, 0, 0, 0, 0, 0, /* ` a b c d e f g */
158 0, 0, 0, 0, 0, 0, 0, 0, /* h i j k l m n o */
159 0, 0, 0, 0, 0, 0, 0, 0, /* p q r s t u v w */
160 0, 0, 0, U, U, U, 0, U, /* x y z { | } ~ DEL */
161
162 U, U, U, U, U, U, U, U, U, U, U, U, U, U, U, U,
163 U, U, U, U, U, U, U, U, U, U, U, U, U, U, U, U,
164 U, U, U, U, U, U, U, U, U, U, U, U, U, U, U, U,
165 U, U, U, U, U, U, U, U, U, U, U, U, U, U, U, U,
166
167 U, U, U, U, U, U, U, U, U, U, U, U, U, U, U, U,
168 U, U, U, U, U, U, U, U, U, U, U, U, U, U, U, U,
169 U, U, U, U, U, U, U, U, U, U, U, U, U, U, U, U,
170 U, U, U, U, U, U, U, U, U, U, U, U, U, U, U, U,
171 };
172 #undef R
173 #undef U
174 #undef RU
175
176 static void
177 url_unescape_1 (char *s, unsigned char mask)
178 {
179 unsigned char *t = (unsigned char *) s; /* t - tortoise */
180 unsigned char *h = (unsigned char *) s; /* h - hare */
181
182 for (; *h; h++, t++)
183 {
184 if (*h != '%')
185 {
186 copychar:
187 *t = *h;
188 }
189 else
190 {
191 unsigned char c;
192 /* Do nothing if '%' is not followed by two hex digits. */
193 if (!h[1] || !h[2] || !(c_isxdigit (h[1]) && c_isxdigit (h[2])))
194 goto copychar;
195 c = X2DIGITS_TO_NUM (h[1], h[2]);
196 if (urlchr_test(c, mask))
197 goto copychar;
198 /* Don't unescape %00 because there is no way to insert it
199 into a C string without effectively truncating it. */
200 if (c == '\0')
201 goto copychar;
202 *t = c;
203 h += 2;
204 }
205 }
206 *t = '\0';
207 }
208
209 /* URL-unescape the string S.
210
211 This is done by transforming the sequences "%HH" to the character
212 represented by the hexadecimal digits HH. If % is not followed by
213 two hexadecimal digits, it is inserted literally.
214
215 The transformation is done in place. If you need the original
216 string intact, make a copy before calling this function. */
217 void
218 url_unescape (char *s)
219 {
220 url_unescape_1 (s, 0);
221 }
222
223 /* URL-unescape the string S.
224
225 This functions behaves identically as url_unescape(), but does not
226 convert characters from "reserved". In other words, it only converts
227 "unsafe" characters. */
228 void
229 url_unescape_except_reserved (char *s)
230 {
231 url_unescape_1 (s, urlchr_reserved);
232 }
233
234 /* The core of url_escape_* functions. Escapes the characters that
235 match the provided mask in urlchr_table.
236
237 If ALLOW_PASSTHROUGH is true, a string with no unsafe chars will be
238 returned unchanged. If ALLOW_PASSTHROUGH is false, a freshly
239 allocated string will be returned in all cases. */
240
241 static char *
242 url_escape_1 (const char *s, unsigned char mask, bool allow_passthrough)
243 {
244 const char *p1;
245 char *p2, *newstr;
246 int newlen;
247 int addition = 0;
248
249 for (p1 = s; *p1; p1++)
250 if (urlchr_test (*p1, mask))
251 addition += 2; /* Two more characters (hex digits) */
252
253 if (!addition)
254 return allow_passthrough ? (char *)s : xstrdup (s);
255
256 newlen = (p1 - s) + addition;
257 newstr = xmalloc (newlen + 1);
258
259 p1 = s;
260 p2 = newstr;
261 while (*p1)
262 {
263 /* Quote the characters that match the test mask. */
264 if (urlchr_test (*p1, mask))
265 {
266 unsigned char c = *p1++;
267 *p2++ = '%';
268 *p2++ = XNUM_TO_DIGIT (c >> 4);
269 *p2++ = XNUM_TO_DIGIT (c & 0xf);
270 }
271 else
272 *p2++ = *p1++;
273 }
274 assert (p2 - newstr == newlen);
275 *p2 = '\0';
276
277 return newstr;
278 }
279
280 /* URL-escape the unsafe characters (see urlchr_table) in a given
281 string, returning a freshly allocated string. */
282
283 char *
284 url_escape (const char *s)
285 {
286 return url_escape_1 (s, urlchr_unsafe, false);
287 }
288
289 /* URL-escape the unsafe and reserved characters (see urlchr_table) in
290 a given string, returning a freshly allocated string. */
291
292 char *
293 url_escape_unsafe_and_reserved (const char *s)
294 {
295 return url_escape_1 (s, urlchr_unsafe|urlchr_reserved, false);
296 }
297
298 /* URL-escape the unsafe characters (see urlchr_table) in a given
299 string. If no characters are unsafe, S is returned. */
300
301 static char *
302 url_escape_allow_passthrough (const char *s)
303 {
304 return url_escape_1 (s, urlchr_unsafe, true);
305 }
306
307 /* Decide whether the char at position P needs to be encoded. (It is
308 not enough to pass a single char *P because the function may need
309 to inspect the surrounding context.)
310
311 Return true if the char should be escaped as %XX, false otherwise. */
312
313 static inline bool
314 char_needs_escaping (const char *p)
315 {
316 if (*p == '%')
317 {
318 if (c_isxdigit (*(p + 1)) && c_isxdigit (*(p + 2)))
319 return false;
320 else
321 /* Garbled %.. sequence: encode `%'. */
322 return true;
323 }
324 else if (URL_UNSAFE_CHAR (*p) && !URL_RESERVED_CHAR (*p))
325 return true;
326 else
327 return false;
328 }
329
330 /* Translate a %-escaped (but possibly non-conformant) input string S
331 into a %-escaped (and conformant) output string. If no characters
332 are encoded or decoded, return the same string S; otherwise, return
333 a freshly allocated string with the new contents.
334
335 After a URL has been run through this function, the protocols that
336 use `%' as the quote character can use the resulting string as-is,
337 while those that don't can use url_unescape to get to the intended
338 data. This function is stable: once the input is transformed,
339 further transformations of the result yield the same output.
340
341 Let's discuss why this function is needed.
342
343 Imagine Wget is asked to retrieve `http://abc.xyz/abc def'. Since
344 a raw space character would mess up the HTTP request, it needs to
345 be quoted, like this:
346
347 GET /abc%20def HTTP/1.0
348
349 It would appear that the unsafe chars need to be quoted, for
350 example with url_escape. But what if we're requested to download
351 `abc%20def'? url_escape transforms "%" to "%25", which would leave
352 us with `abc%2520def'. This is incorrect -- since %-escapes are
353 part of URL syntax, "%20" is the correct way to denote a literal
354 space on the Wget command line. This leads to the conclusion that
355 in that case Wget should not call url_escape, but leave the `%20'
356 as is. This is clearly contradictory, but it only gets worse.
357
358 What if the requested URI is `abc%20 def'? If we call url_escape,
359 we end up with `/abc%2520%20def', which is almost certainly not
360 intended. If we don't call url_escape, we are left with the
361 embedded space and cannot complete the request. What the user
362 meant was for Wget to request `/abc%20%20def', and this is where
363 reencode_escapes kicks in.
364
365 Wget used to solve this by first decoding %-quotes, and then
366 encoding all the "unsafe" characters found in the resulting string.
367 This was wrong because it didn't preserve certain URL special
368 (reserved) characters. For instance, URI containing "a%2B+b" (0x2b
369 == '+') would get translated to "a%2B%2Bb" or "a++b" depending on
370 whether we considered `+' reserved (it is). One of these results
371 is inevitable because by the second step we would lose information
372 on whether the `+' was originally encoded or not. Both results
373 were wrong because in CGI parameters + means space, while %2B means
374 literal plus. reencode_escapes correctly translates the above to
375 "a%2B+b", i.e. returns the original string.
376
377 This function uses a modified version of the algorithm originally
378 proposed by Anon Sricharoenchai:
379
380 * Encode all "unsafe" characters, except those that are also
381 "reserved", to %XX. See urlchr_table for which characters are
382 unsafe and reserved.
383
384 * Encode the "%" characters not followed by two hex digits to
385 "%25".
386
387 * Pass through all other characters and %XX escapes as-is. (Up to
388 Wget 1.10 this decoded %XX escapes corresponding to "safe"
389 characters, but that was obtrusive and broke some servers.)
390
391 Anon's test case:
392
393 "http://abc.xyz/%20%3F%%36%31%25aa% a?a=%61+a%2Ba&b=b%26c%3Dc"
394 ->
395 "http://abc.xyz/%20%3F%25%36%31%25aa%25%20a?a=%61+a%2Ba&b=b%26c%3Dc"
396
397 Simpler test cases:
398
399 "foo bar" -> "foo%20bar"
400 "foo%20bar" -> "foo%20bar"
401 "foo %20bar" -> "foo%20%20bar"
402 "foo%%20bar" -> "foo%25%20bar" (0x25 == '%')
403 "foo%25%20bar" -> "foo%25%20bar"
404 "foo%2%20bar" -> "foo%252%20bar"
405 "foo+bar" -> "foo+bar" (plus is reserved!)
406 "foo%2b+bar" -> "foo%2b+bar" */
407
408 static char *
409 reencode_escapes (const char *s)
410 {
411 const char *p1;
412 char *newstr, *p2;
413 int oldlen, newlen;
414
415 int encode_count = 0;
416
417 /* First pass: inspect the string to see if there's anything to do,
418 and to calculate the new length. */
419 for (p1 = s; *p1; p1++)
420 if (char_needs_escaping (p1))
421 ++encode_count;
422
423 if (!encode_count)
424 /* The string is good as it is. */
425 return (char *) s; /* C const model sucks. */
426
427 oldlen = p1 - s;
428 /* Each encoding adds two characters (hex digits). */
429 newlen = oldlen + 2 * encode_count;
430 newstr = xmalloc (newlen + 1);
431
432 /* Second pass: copy the string to the destination address, encoding
433 chars when needed. */
434 p1 = s;
435 p2 = newstr;
436
437 while (*p1)
438 if (char_needs_escaping (p1))
439 {
440 unsigned char c = *p1++;
441 *p2++ = '%';
442 *p2++ = XNUM_TO_DIGIT (c >> 4);
443 *p2++ = XNUM_TO_DIGIT (c & 0xf);
444 }
445 else
446 *p2++ = *p1++;
447
448 *p2 = '\0';
449 assert (p2 - newstr == newlen);
450 return newstr;
451 }
452
453 /* Returns the scheme type if the scheme is supported, or
454 SCHEME_INVALID if not. */
455
456 enum url_scheme
457 url_scheme (const char *url)
458 {
459 int i;
460
461 for (i = 0; supported_schemes[i].leading_string; i++)
462 if (0 == c_strncasecmp (url, supported_schemes[i].leading_string,
463 strlen (supported_schemes[i].leading_string)))
464 {
465 if (!(supported_schemes[i].flags & scm_disabled))
466 return (enum url_scheme) i;
467 else
468 return SCHEME_INVALID;
469 }
470
471 return SCHEME_INVALID;
472 }
473
474 #define SCHEME_CHAR(ch) (c_isalnum (ch) || (ch) == '-' || (ch) == '+')
475
476 /* Return 1 if the URL begins with any "scheme", 0 otherwise. As
477 currently implemented, it returns true if URL begins with
478 [-+a-zA-Z0-9]+: . */
479
480 bool
481 url_has_scheme (const char *url)
482 {
483 const char *p = url;
484
485 /* The first char must be a scheme char. */
486 if (!*p || !SCHEME_CHAR (*p))
487 return false;
488 ++p;
489 /* Followed by 0 or more scheme chars. */
490 while (*p && SCHEME_CHAR (*p))
491 ++p;
492 /* Terminated by ':'. */
493 return *p == ':';
494 }
495
496 bool
497 url_valid_scheme (const char *url)
498 {
499 enum url_scheme scheme = url_scheme (url);
500 return scheme != SCHEME_INVALID;
501 }
502
503 int
504 scheme_default_port (enum url_scheme scheme)
505 {
506 return supported_schemes[scheme].default_port;
507 }
508
509 void
510 scheme_disable (enum url_scheme scheme)
511 {
512 supported_schemes[scheme].flags |= scm_disabled;
513 }
514
515 const char *
516 scheme_leading_string (enum url_scheme scheme)
517 {
518 return supported_schemes[scheme].leading_string;
519 }
520
521 /* Skip the username and password, if present in the URL. The
522 function should *not* be called with the complete URL, but with the
523 portion after the scheme.
524
525 If no username and password are found, return URL. */
526
527 static const char *
528 url_skip_credentials (const char *url)
529 {
530 /*
531 * This whole file implements https://www.rfc-editor.org/rfc/rfc2396 .
532 * RFC 2396 is outdated since 2005 and needs a rewrite or a thorough re-visit.
533 *
534 * The RFC says
535 * server = [ [ userinfo "@" ] hostport ]
536 * userinfo = *( unreserved | escaped | ";" | ":" | "&" | "=" | "+" | "$" | "," )
537 * unreserved = alphanum | mark
538 * mark = "-" | "_" | "." | "!" | "~" | "*" | "'" | "(" | ")"
539 */
540 static const char *allowed = "-_.!~*'();:&=+$,";
541
542 for (const char *p = url; *p; p++)
543 {
544 if (c_isalnum(*p))
545 continue;
546
547 if (strchr(allowed, *p))
548 continue;
549
550 if (*p == '%' && c_isxdigit(p[1]) && c_isxdigit(p[2]))
551 {
552 p += 2;
553 continue;
554 }
555
556 if (*p == '@')
557 return p + 1;
558
559 break;
560 }
561
562 return url;
563 }
564
565 /* Parse credentials contained in [BEG, END). The region is expected
566 to have come from a URL and is unescaped. */
567
568 static bool
569 parse_credentials (const char *beg, const char *end, char **user, char **passwd)
570 {
571 char *colon;
572 const char *userend;
573
574 if (beg == end)
575 return false; /* empty user name */
576
577 colon = memchr (beg, ':', end - beg);
578 if (colon == beg)
579 return false; /* again empty user name */
580
581 if (colon)
582 {
583 *passwd = strdupdelim (colon + 1, end);
584 userend = colon;
585 url_unescape (*passwd);
586 }
587 else
588 {
589 *passwd = NULL;
590 userend = end;
591 }
592 *user = strdupdelim (beg, userend);
593 url_unescape (*user);
594 return true;
595 }
596
597 static bool is_valid_port(const char *p)
598 {
599 unsigned port = (unsigned) atoi (p);
600 if (port == 0 || port > 65535)
601 return false;
602
603 int digits = strspn (p, "0123456789");
604 return digits && (p[digits] == '/' || p[digits] == '\0');
605 }
606
607 /* Prepend "http://" to url if scheme is missing, otherwise return NULL. */
608 char *
609 maybe_prepend_scheme (const char *url)
610 {
611 if (url_scheme (url) != SCHEME_INVALID)
612 return NULL;
613
614 const char *p = strchr (url, ':');
615 if (p == url)
616 return NULL;
617
618 /* If we're looking at "://", it means the URL uses a scheme we
619 don't support, which may include "https" when compiled without
620 SSL support. Don't bogusly prepend "http://" to such URLs. */
621 if (p && p[0] == ':' && p[1] == '/' && p[2] == '/')
622 return NULL;
623
624 if (p && p[0] == ':' && !is_valid_port (p + 1))
625 return NULL;
626
627
628 fprintf(stderr, "Prepended http:// to '%s'\n", url);
629 return aprintf ("http://%s", url);
630 }
631
632 static void split_path (const char *, char **, char **);
633
634 /* Like strpbrk, with the exception that it returns the pointer to the
635 terminating zero (end-of-string aka "eos") if no matching character
636 is found. */
637
638 static inline char *
639 strpbrk_or_eos (const char *s, const char *accept)
640 {
641 char *p = strpbrk (s, accept);
642 if (!p)
643 p = strchr (s, '\0');
644 return p;
645 }
646
647 /* Turn STR into lowercase; return true if a character was actually
648 changed. */
649
650 static bool
651 lowercase_str (char *str)
652 {
653 bool changed = false;
654 for (; *str; str++)
655 if (c_isupper (*str))
656 {
657 changed = true;
658 *str = c_tolower (*str);
659 }
660 return changed;
661 }
662
663 static const char *
664 init_seps (enum url_scheme scheme)
665 {
666 static char seps[8] = ":/";
667 char *p = seps + 2;
668 int flags = supported_schemes[scheme].flags;
669
670 if (flags & scm_has_params)
671 *p++ = ';';
672 if (flags & scm_has_query)
673 *p++ = '?';
674 if (flags & scm_has_fragment)
675 *p++ = '#';
676 *p = '\0';
677 return seps;
678 }
679
680 enum {
681 PE_NO_ERROR = 0,
682 PE_UNSUPPORTED_SCHEME,
683 PE_UNSUPPORTED_SCHEME_HTTPS,
684 PE_UNSUPPORTED_SCHEME_FTPS,
685 PE_MISSING_SCHEME,
686 PE_INVALID_HOST_NAME,
687 PE_BAD_PORT_NUMBER,
688 PE_INVALID_USER_NAME,
689 PE_UNTERMINATED_IPV6_ADDRESS,
690 PE_IPV6_NOT_SUPPORTED,
691 PE_INVALID_IPV6_ADDRESS
692 };
693
694 static const char *parse_errors[] = {
695 [PE_NO_ERROR] = N_("No error"),
696 [PE_UNSUPPORTED_SCHEME] = N_("Unsupported scheme"),
697 [PE_UNSUPPORTED_SCHEME_HTTPS] = N_("HTTPS support not compiled in"),
698 [PE_UNSUPPORTED_SCHEME_FTPS] = N_("FTPS support not compiled in"),
699 [PE_MISSING_SCHEME] = N_("Scheme missing"),
700 [PE_INVALID_HOST_NAME] = N_("Invalid host name"),
701 [PE_BAD_PORT_NUMBER] = N_("Bad port number"),
702 [PE_INVALID_USER_NAME] = N_("Invalid user name"),
703 [PE_UNTERMINATED_IPV6_ADDRESS] = N_("Unterminated IPv6 numeric address"),
704 [PE_IPV6_NOT_SUPPORTED] = N_("IPv6 addresses not supported"),
705 [PE_INVALID_IPV6_ADDRESS] = N_("Invalid IPv6 numeric address")
706 };
707
708 /* Parse a URL.
709
710 Return a new struct url if successful, NULL on error. In case of
711 error, and if ERROR is not NULL, also set *ERROR to the appropriate
712 error code. */
713 struct url *
714 url_parse (const char *url, int *error, struct iri *iri, bool percent_encode)
715 {
716 struct url *u;
717 const char *p;
718 bool path_modified, host_modified;
719
720 enum url_scheme scheme;
721 const char *seps;
722
723 const char *uname_b, *uname_e;
724 const char *host_b, *host_e;
725 const char *path_b, *path_e;
726 const char *params_b, *params_e;
727 const char *query_b, *query_e;
728 const char *fragment_b, *fragment_e;
729
730 int port;
731 char *user = NULL, *passwd = NULL;
732
733 const char *url_encoded = NULL;
734
735 int error_code;
736
737 scheme = url_scheme (url);
738 if (scheme == SCHEME_INVALID)
739 {
740 if (!url_has_scheme (url))
741 error_code = PE_MISSING_SCHEME;
742 else if (!c_strncasecmp (url, "https:", 6))
743 error_code = PE_UNSUPPORTED_SCHEME_HTTPS;
744 else if (!c_strncasecmp (url, "ftps:", 5))
745 error_code = PE_UNSUPPORTED_SCHEME_FTPS;
746 else
747 error_code = PE_UNSUPPORTED_SCHEME;
748 goto error;
749 }
750
751 url_encoded = url;
752
753 if (iri && iri->utf8_encode)
754 {
755 char *new_url = NULL;
756
757 iri->utf8_encode = remote_to_utf8 (iri, iri->orig_url ? iri->orig_url : url, &new_url);
758 if (!iri->utf8_encode)
759 new_url = NULL;
760 else
761 {
762 xfree (iri->orig_url);
763 iri->orig_url = xstrdup (url);
764 url_encoded = reencode_escapes (new_url);
765 if (url_encoded != new_url)
766 xfree (new_url);
767 percent_encode = false;
768 }
769 }
770
771 if (percent_encode)
772 url_encoded = reencode_escapes (url);
773
774 p = url_encoded;
775 p += strlen (supported_schemes[scheme].leading_string);
776 uname_b = p;
777 p = url_skip_credentials (p);
778 uname_e = p;
779
780 /* scheme://user:pass@host[:port]... */
781 /* ^ */
782
783 /* We attempt to break down the URL into the components path,
784 params, query, and fragment. They are ordered like this:
785
786 scheme://host[:port][/path][;params][?query][#fragment] */
787
788 path_b = path_e = NULL;
789 params_b = params_e = NULL;
790 query_b = query_e = NULL;
791 fragment_b = fragment_e = NULL;
792
793 /* Initialize separators for optional parts of URL, depending on the
794 scheme. For example, FTP has params, and HTTP and HTTPS have
795 query string and fragment. */
796 seps = init_seps (scheme);
797
798 host_b = p;
799
800 if (*p == '[')
801 {
802 /* Handle IPv6 address inside square brackets. Ideally we'd
803 just look for the terminating ']', but rfc2732 mandates
804 rejecting invalid IPv6 addresses. */
805
806 /* The address begins after '['. */
807 host_b = p + 1;
808 host_e = strchr (host_b, ']');
809
810 if (!host_e)
811 {
812 error_code = PE_UNTERMINATED_IPV6_ADDRESS;
813 goto error;
814 }
815
816 #ifdef ENABLE_IPV6
817 /* Check if the IPv6 address is valid. */
818 if (!is_valid_ipv6_address(host_b, host_e))
819 {
820 error_code = PE_INVALID_IPV6_ADDRESS;
821 goto error;
822 }
823
824 /* Continue parsing after the closing ']'. */
825 p = host_e + 1;
826 #else
827 error_code = PE_IPV6_NOT_SUPPORTED;
828 goto error;
829 #endif
830
831 /* The closing bracket must be followed by a separator or by the
832 null char. */
833 /* http://[::1]... */
834 /* ^ */
835 if (!strchr (seps, *p))
836 {
837 /* Trailing garbage after []-delimited IPv6 address. */
838 error_code = PE_INVALID_HOST_NAME;
839 goto error;
840 }
841 }
842 else
843 {
844 p = strpbrk_or_eos (p, seps);
845 host_e = p;
846 }
847 ++seps; /* advance to '/' */
848
849 if (host_b == host_e)
850 {
851 error_code = PE_INVALID_HOST_NAME;
852 goto error;
853 }
854
855 port = scheme_default_port (scheme);
856 if (*p == ':')
857 {
858 const char *port_b, *port_e, *pp;
859
860 /* scheme://host:port/tralala */
861 /* ^ */
862 ++p;
863 port_b = p;
864 p = strpbrk_or_eos (p, seps);
865 port_e = p;
866
867 /* Allow empty port, as per rfc2396. */
868 if (port_b != port_e)
869 for (port = 0, pp = port_b; pp < port_e; pp++)
870 {
871 if (!c_isdigit (*pp))
872 {
873 /* http://host:12randomgarbage/blah */
874 /* ^ */
875 error_code = PE_BAD_PORT_NUMBER;
876 goto error;
877 }
878 port = 10 * port + (*pp - '0');
879 /* Check for too large port numbers here, before we have
880 a chance to overflow on bogus port values. */
881 if (port > 0xffff)
882 {
883 error_code = PE_BAD_PORT_NUMBER;
884 goto error;
885 }
886 }
887 }
888 /* Advance to the first separator *after* '/' (either ';' or '?',
889 depending on the scheme). */
890 ++seps;
891
892 /* Get the optional parts of URL, each part being delimited by
893 current location and the position of the next separator. */
894 #define GET_URL_PART(sepchar, var) do { \
895 if (*p == sepchar) \
896 var##_b = ++p, var##_e = p = strpbrk_or_eos (p, seps); \
897 ++seps; \
898 } while (0)
899
900 GET_URL_PART ('/', path);
901 if (supported_schemes[scheme].flags & scm_has_params)
902 GET_URL_PART (';', params);
903 if (supported_schemes[scheme].flags & scm_has_query)
904 GET_URL_PART ('?', query);
905 if (supported_schemes[scheme].flags & scm_has_fragment)
906 GET_URL_PART ('#', fragment);
907
908 #undef GET_URL_PART
909 assert (*p == 0);
910
911 if (uname_b != uname_e)
912 {
913 /* http://user:pass@host */
914 /* ^ ^ */
915 /* uname_b uname_e */
916 if (!parse_credentials (uname_b, uname_e - 1, &user, &passwd))
917 {
918 error_code = PE_INVALID_USER_NAME;
919 goto error;
920 }
921 }
922
923 u = xnew0 (struct url);
924 u->scheme = scheme;
925 u->host = strdupdelim (host_b, host_e);
926 u->port = port;
927 u->user = user;
928 u->passwd = passwd;
929
930 u->path = strdupdelim (path_b, path_e);
931 path_modified = path_simplify (scheme, u->path);
932 split_path (u->path, &u->dir, &u->file);
933
934 host_modified = lowercase_str (u->host);
935
936 /* Decode %HH sequences in host name. This is important not so much
937 to support %HH sequences in host names (which other browser
938 don't), but to support binary characters (which will have been
939 converted to %HH by reencode_escapes). */
940 if (strchr (u->host, '%'))
941 {
942 url_unescape (u->host);
943 host_modified = true;
944
945 /* check for invalid control characters in host name */
946 for (p = u->host; *p; p++)
947 {
948 if (c_iscntrl(*p))
949 {
950 url_free(u);
951 error_code = PE_INVALID_HOST_NAME;
952 goto error;
953 }
954 }
955
956 /* Apply IDNA regardless of iri->utf8_encode status */
957 if (opt.enable_iri && iri)
958 {
959 char *new = idn_encode (iri, u->host);
960 if (new)
961 {
962 xfree (u->host);
963 u->host = new;
964 host_modified = true;
965 }
966 }
967 }
968
969 if (params_b)
970 u->params = strdupdelim (params_b, params_e);
971 if (query_b)
972 u->query = strdupdelim (query_b, query_e);
973 if (fragment_b)
974 u->fragment = strdupdelim (fragment_b, fragment_e);
975
976 if (opt.enable_iri || path_modified || u->fragment || host_modified || path_b == path_e)
977 {
978 /* If we suspect that a transformation has rendered what
979 url_string might return different from URL_ENCODED, rebuild
980 u->url using url_string. */
981 u->url = url_string (u, URL_AUTH_SHOW);
982
983 if (url_encoded != url)
984 xfree (url_encoded);
985 }
986 else
987 {
988 if (url_encoded == url)
989 u->url = xstrdup (url);
990 else
991 u->url = (char *) url_encoded;
992 }
993
994 return u;
995
996 error:
997 /* Cleanup in case of error: */
998 if (url_encoded && url_encoded != url)
999 xfree (url_encoded);
1000
1001 /* Transmit the error code to the caller, if the caller wants to
1002 know. */
1003 if (error)
1004 *error = error_code;
1005 return NULL;
1006 }
1007
1008 /* Return the error message string from ERROR_CODE, which should have
1009 been retrieved from url_parse. The error message is translated. */
1010
1011 const char *
1012 url_error (int error_code)
1013 {
1014 assert (error_code >= 0 && error_code < (int) countof (parse_errors));
1015
1016 if (error_code >= 0 && error_code < (int) countof (parse_errors))
1017 return _(parse_errors[error_code]);
1018
1019 return ""; // This should never be reached
1020 }
1021
1022 /* Split PATH into DIR and FILE. PATH comes from the URL and is
1023 expected to be URL-escaped.
1024
1025 The path is split into directory (the part up to the last slash)
1026 and file (the part after the last slash), which are subsequently
1027 unescaped. Examples:
1028
1029 PATH DIR FILE
1030 "foo/bar/baz" "foo/bar" "baz"
1031 "foo/bar/" "foo/bar" ""
1032 "foo" "" "foo"
1033 "foo/bar/baz%2fqux" "foo/bar" "baz/qux" (!)
1034
1035 DIR and FILE are freshly allocated. */
1036
1037 static void
1038 split_path (const char *path, char **dir, char **file)
1039 {
1040 char *last_slash = strrchr (path, '/');
1041 if (!last_slash)
1042 {
1043 *dir = xstrdup ("");
1044 *file = xstrdup (path);
1045 }
1046 else
1047 {
1048 *dir = strdupdelim (path, last_slash);
1049 *file = xstrdup (last_slash + 1);
1050 }
1051 url_unescape (*dir);
1052 url_unescape (*file);
1053 }
1054
1055 /* Note: URL's "full path" is the path with the query string and
1056 params appended. The "fragment" (#foo) is intentionally ignored,
1057 but that might be changed. For example, if the original URL was
1058 "http://host:port/foo/bar/baz;bullshit?querystring#uselessfragment",
1059 the full path will be "/foo/bar/baz;bullshit?querystring". */
1060
1061 /* Return the length of the full path, without the terminating
1062 zero. */
1063
1064 static int
1065 full_path_length (const struct url *url)
1066 {
1067 int len = 0;
1068
1069 #define FROB(el) if (url->el) len += 1 + strlen (url->el)
1070
1071 FROB (path);
1072 FROB (params);
1073 FROB (query);
1074
1075 #undef FROB
1076
1077 return len;
1078 }
1079
1080 /* Write out the full path. */
1081
1082 static void
1083 full_path_write (const struct url *url, char *where)
1084 {
1085 #define FROB(el, chr) do { \
1086 char *f_el = url->el; \
1087 if (f_el) { \
1088 int l = strlen (f_el); \
1089 *where++ = chr; \
1090 memcpy (where, f_el, l); \
1091 where += l; \
1092 } \
1093 } while (0)
1094
1095 FROB (path, '/');
1096 FROB (params, ';');
1097 FROB (query, '?');
1098
1099 #undef FROB
1100 }
1101
1102 /* Public function for getting the "full path". E.g. if u->path is
1103 "foo/bar" and u->query is "param=value", full_path will be
1104 "/foo/bar?param=value". */
1105
1106 char *
1107 url_full_path (const struct url *url)
1108 {
1109 int length = full_path_length (url);
1110 char *full_path = xmalloc (length + 1);
1111
1112 full_path_write (url, full_path);
1113 full_path[length] = '\0';
1114
1115 return full_path;
1116 }
1117
1118 /* Unescape CHR in an otherwise escaped STR. Used to selectively
1119 escaping of certain characters, such as "/" and ":". Returns a
1120 count of unescaped chars. */
1121
1122 static void
1123 unescape_single_char (char *str, char chr)
1124 {
1125 const char c1 = XNUM_TO_DIGIT (chr >> 4);
1126 const char c2 = XNUM_TO_DIGIT (chr & 0xf);
1127 char *h = str; /* hare */
1128 char *t = str; /* tortoise */
1129 for (; *h; h++, t++)
1130 {
1131 if (h[0] == '%' && h[1] == c1 && h[2] == c2)
1132 {
1133 *t = chr;
1134 h += 2;
1135 }
1136 else
1137 *t = *h;
1138 }
1139 *t = '\0';
1140 }
1141
1142 /* Escape unsafe and reserved characters, except for the slash
1143 characters. */
1144
1145 static char *
1146 url_escape_dir (const char *dir)
1147 {
1148 char *newdir = url_escape_1 (dir, urlchr_unsafe | urlchr_reserved, 1);
1149 if (newdir == dir)
1150 return (char *)dir;
1151
1152 unescape_single_char (newdir, '/');
1153 return newdir;
1154 }
1155
1156 /* Sync u->path and u->url with u->dir and u->file. Called after
1157 u->file or u->dir have been changed, typically by the FTP code. */
1158
1159 static void
1160 sync_path (struct url *u)
1161 {
1162 char *newpath, *efile, *edir;
1163
1164 xfree (u->path);
1165
1166 /* u->dir and u->file are not escaped. URL-escape them before
1167 reassembling them into u->path. That way, if they contain
1168 separators like '?' or even if u->file contains slashes, the
1169 path will be correctly assembled. (u->file can contain slashes
1170 if the URL specifies it with %2f, or if an FTP server returns
1171 it.) */
1172 edir = url_escape_dir (u->dir);
1173 efile = url_escape_1 (u->file, urlchr_unsafe | urlchr_reserved, 1);
1174
1175 if (!*edir)
1176 newpath = xstrdup (efile);
1177 else
1178 {
1179 int dirlen = strlen (edir);
1180 int filelen = strlen (efile);
1181
1182 /* Copy "DIR/FILE" to newpath. */
1183 char *p = newpath = xmalloc (dirlen + 1 + filelen + 1);
1184 memcpy (p, edir, dirlen);
1185 p += dirlen;
1186 *p++ = '/';
1187 memcpy (p, efile, filelen);
1188 p += filelen;
1189 *p = '\0';
1190 }
1191
1192 u->path = newpath;
1193
1194 if (edir != u->dir)
1195 xfree (edir);
1196 if (efile != u->file)
1197 xfree (efile);
1198
1199 /* Regenerate u->url as well. */
1200 xfree (u->url);
1201 u->url = url_string (u, URL_AUTH_SHOW);
1202 }
1203
1204 /* Mutators. Code in ftp.c insists on changing u->dir and u->file.
1205 This way we can sync u->path and u->url when they get changed. */
1206
1207 void
1208 url_set_dir (struct url *url, const char *newdir)
1209 {
1210 xfree (url->dir);
1211 url->dir = xstrdup (newdir);
1212 sync_path (url);
1213 }
1214
1215 void
1216 url_set_file (struct url *url, const char *newfile)
1217 {
1218 xfree (url->file);
1219 url->file = xstrdup (newfile);
1220 sync_path (url);
1221 }
1222
1223 void
1224 url_free (struct url *url)
1225 {
1226 if (url)
1227 {
1228 xfree (url->host);
1229
1230 xfree (url->path);
1231 xfree (url->url);
1232
1233 xfree (url->params);
1234 xfree (url->query);
1235 xfree (url->fragment);
1236 xfree (url->user);
1237 xfree (url->passwd);
1238
1239 xfree (url->dir);
1240 xfree (url->file);
1241
1242 xfree (url);
1243 }
1244 }
1245
1246 /* Create all the necessary directories for PATH (a file). Calls
1247 make_directory internally. */
1248 int
1249 mkalldirs (const char *path)
1250 {
1251 const char *p;
1252 char *t;
1253 struct stat st;
1254 int res;
1255
1256 p = strrchr(path, '/');
1257 p = p == NULL ? path : p;
1258
1259 /* Don't create if it's just a file. */
1260 if ((p == path) && (*p != '/'))
1261 return 0;
1262 t = strdupdelim (path, p);
1263
1264 /* Check whether the directory exists. */
1265 if ((stat (t, &st) == 0))
1266 {
1267 if (S_ISDIR (st.st_mode))
1268 {
1269 xfree (t);
1270 return 0;
1271 }
1272 else
1273 {
1274 /* If the dir exists as a file name, remove it first. This
1275 is *only* for Wget to work with buggy old CERN http
1276 servers. Here is the scenario: When Wget tries to
1277 retrieve a directory without a slash, e.g.
1278 http://foo/bar (bar being a directory), CERN server will
1279 not redirect it too http://foo/bar/ -- it will generate a
1280 directory listing containing links to bar/file1,
1281 bar/file2, etc. Wget will lose because it saves this
1282 HTML listing to a file `bar', so it cannot create the
1283 directory. To work around this, if the file of the same
1284 name exists, we just remove it and create the directory
1285 anyway. */
1286 DEBUGP (("Removing %s because of directory danger!\n", t));
1287 if (unlink (t))
1288 logprintf (LOG_NOTQUIET, "Failed to unlink %s (%d): %s\n",
1289 t, errno, strerror(errno));
1290 }
1291 }
1292 res = make_directory (t);
1293 if (res != 0)
1294 logprintf (LOG_NOTQUIET, "%s: %s\n", t, strerror (errno));
1295 xfree (t);
1296 return res;
1297 }
1298
1299 /* Functions for constructing the file name out of URL components. */
1300
1301 /* A growable string structure, used by url_file_name and friends.
1302 This should perhaps be moved to utils.c.
1303
1304 The idea is to have a convenient and efficient way to construct a
1305 string by having various functions append data to it. Instead of
1306 passing the obligatory BASEVAR, SIZEVAR and TAILPOS to all the
1307 functions in questions, we pass the pointer to this struct.
1308
1309 Functions that write to the members in this struct must make sure
1310 that base remains null terminated by calling append_null().
1311 */
1312
1313 struct growable {
1314 char *base;
1315 int size; /* memory allocated */
1316 int tail; /* string length */
1317 };
1318
1319 /* Ensure that the string can accept APPEND_COUNT more characters past
1320 the current TAIL position. If necessary, this will grow the string
1321 and update its allocated size. If the string is already large
1322 enough to take TAIL+APPEND_COUNT characters, this does nothing. */
1323 #define GROW(g, append_size) do { \
1324 struct growable *G_ = g; \
1325 DO_REALLOC (G_->base, G_->size, G_->tail + append_size, char); \
1326 } while (0)
1327
1328 /* Return the tail position of the string. */
1329 #define TAIL(r) ((r)->base + (r)->tail)
1330
1331 /* Move the tail position by APPEND_COUNT characters. */
1332 #define TAIL_INCR(r, append_count) ((r)->tail += append_count)
1333
1334
1335 /* Append NULL to DEST. */
1336 static void
1337 append_null (struct growable *dest)
1338 {
1339 GROW (dest, 1);
1340 *TAIL (dest) = 0;
1341 }
1342
1343 /* Append CH to DEST. */
1344 static void
1345 append_char (char ch, struct growable *dest)
1346 {
1347 if (ch)
1348 {
1349 GROW (dest, 1);
1350 *TAIL (dest) = ch;
1351 TAIL_INCR (dest, 1);
1352 }
1353
1354 append_null (dest);
1355 }
1356
1357 /* Append the string STR to DEST. */
1358 static void
1359 append_string (const char *str, struct growable *dest)
1360 {
1361 int l = strlen (str);
1362
1363 if (l)
1364 {
1365 GROW (dest, l);
1366 memcpy (TAIL (dest), str, l);
1367 TAIL_INCR (dest, l);
1368 }
1369
1370 append_null (dest);
1371 }
1372
1373
1374 enum {
1375 filechr_not_unix = 1, /* unusable on Unix, / and \0 */
1376 filechr_not_vms = 2, /* unusable on VMS (ODS5), 0x00-0x1F * ? */
1377 filechr_not_windows = 4, /* unusable on Windows, one of \|/<>?:*" */
1378 filechr_control = 8 /* a control character, e.g. 0-31 */
1379 };
1380
1381 #define FILE_CHAR_TEST(c, mask) \
1382 ((opt.restrict_files_nonascii && !c_isascii ((unsigned char)(c))) || \
1383 (filechr_table[(unsigned char)(c)] & (mask)))
1384
1385 /* Shorthands for the table: */
1386 #define U filechr_not_unix
1387 #define V filechr_not_vms
1388 #define W filechr_not_windows
1389 #define C filechr_control
1390
1391 #define UVWC U|V|W|C
1392 #define UW U|W
1393 #define VC V|C
1394 #define VW V|W
1395
1396 /* Table of characters unsafe under various conditions (see above).
1397
1398 Arguably we could also claim `%' to be unsafe, since we use it as
1399 the escape character. If we ever want to be able to reliably
1400 translate file name back to URL, this would become important
1401 crucial. Right now, it's better to be minimal in escaping. */
1402
1403 static const unsigned char filechr_table[256] =
1404 {
1405 UVWC, VC, VC, VC, VC, VC, VC, VC, /* NUL SOH STX ETX EOT ENQ ACK BEL */
1406 VC, VC, VC, VC, VC, VC, VC, VC, /* BS HT LF VT FF CR SO SI */
1407 VC, VC, VC, VC, VC, VC, VC, VC, /* DLE DC1 DC2 DC3 DC4 NAK SYN ETB */
1408 VC, VC, VC, VC, VC, VC, VC, VC, /* CAN EM SUB ESC FS GS RS US */
1409 0, 0, W, 0, 0, 0, 0, 0, /* SP ! " # $ % & ' */
1410 0, 0, VW, 0, 0, 0, 0, UW, /* ( ) * + , - . / */
1411 0, 0, 0, 0, 0, 0, 0, 0, /* 0 1 2 3 4 5 6 7 */
1412 0, 0, W, 0, W, 0, W, VW, /* 8 9 : ; < = > ? */
1413 0, 0, 0, 0, 0, 0, 0, 0, /* @ A B C D E F G */
1414 0, 0, 0, 0, 0, 0, 0, 0, /* H I J K L M N O */
1415 0, 0, 0, 0, 0, 0, 0, 0, /* P Q R S T U V W */
1416 0, 0, 0, 0, W, 0, 0, 0, /* X Y Z [ \ ] ^ _ */
1417 0, 0, 0, 0, 0, 0, 0, 0, /* ` a b c d e f g */
1418 0, 0, 0, 0, 0, 0, 0, 0, /* h i j k l m n o */
1419 0, 0, 0, 0, 0, 0, 0, 0, /* p q r s t u v w */
1420 0, 0, 0, 0, W, 0, 0, C, /* x y z { | } ~ DEL */
1421
1422 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 128-143 */
1423 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 144-159 */
1424 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
1425 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
1426
1427 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
1428 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
1429 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
1430 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
1431 };
1432 #undef U
1433 #undef V
1434 #undef W
1435 #undef C
1436 #undef UW
1437 #undef UVWC
1438 #undef VC
1439 #undef VW
1440
1441 /* FN_PORT_SEP is the separator between host and port in file names
1442 for non-standard port numbers. On Unix this is normally ':', as in
1443 "www.xemacs.org:4001/index.html". Under Windows, we set it to +
1444 because Windows can't handle ':' in file names. */
1445 #define FN_PORT_SEP (opt.restrict_files_os != restrict_windows ? ':' : '+')
1446
1447 /* FN_QUERY_SEP is the separator between the file name and the URL
1448 query, normally '?'. Because VMS and Windows cannot handle '?' in a
1449 file name, we use '@' instead there. */
1450 #define FN_QUERY_SEP \
1451 (((opt.restrict_files_os != restrict_vms) && \
1452 (opt.restrict_files_os != restrict_windows)) ? '?' : '@')
1453 #define FN_QUERY_SEP_STR \
1454 (((opt.restrict_files_os != restrict_vms) && \
1455 (opt.restrict_files_os != restrict_windows)) ? "?" : "@")
1456
1457 /* Quote path element, characters in [b, e), as file name, and append
1458 the quoted string to DEST. Each character is quoted as per
1459 file_unsafe_char and the corresponding table.
1460
1461 If ESCAPED is true, the path element is considered to be
1462 URL-escaped and will be unescaped prior to inspection. */
1463
1464 static void
1465 append_uri_pathel (const char *b, const char *e, bool escaped,
1466 struct growable *dest)
1467 {
1468 const char *p;
1469 char buf[1024];
1470 char *unescaped = NULL;
1471 int quoted, outlen;
1472 int mask;
1473 int max_length;
1474
1475 if (!dest)
1476 return;
1477
1478 if (opt.restrict_files_os == restrict_unix)
1479 mask = filechr_not_unix;
1480 else if (opt.restrict_files_os == restrict_vms)
1481 mask = filechr_not_vms;
1482 else
1483 mask = filechr_not_windows;
1484
1485 if (opt.restrict_files_ctrl)
1486 mask |= filechr_control;
1487
1488 /* Copy [b, e) to PATHEL and URL-unescape it. */
1489 if (escaped)
1490 {
1491 size_t len = e - b;
1492 if (len < sizeof (buf))
1493 unescaped = buf;
1494 else
1495 unescaped = xmalloc(len + 1);
1496
1497 memcpy(unescaped, b, len);
1498 unescaped[len] = 0;
1499
1500 url_unescape (unescaped);
1501 b = unescaped;
1502 e = unescaped + strlen (unescaped);
1503 }
1504
1505 /* Defang ".." when found as component of path. Remember that path
1506 comes from the URL and might contain malicious input. */
1507 if (e - b == 2 && b[0] == '.' && b[1] == '.')
1508 {
1509 b = "%2E%2E";
1510 e = b + 6;
1511 }
1512
1513 /* Walk the PATHEL string and check how many characters we'll need
1514 to quote. */
1515 quoted = 0;
1516 for (p = b; p < e; p++)
1517 if (FILE_CHAR_TEST (*p, mask))
1518 ++quoted;
1519
1520 /* Calculate the length of the output string. e-b is the input
1521 string length. Each quoted char introduces two additional
1522 characters in the string, hence 2*quoted. */
1523 outlen = (e - b) + (2 * quoted);
1524 # ifdef WINDOWS
1525 max_length = MAX_PATH;
1526 # else
1527 max_length = get_max_length(dest->base, dest->tail, _PC_NAME_MAX);
1528 # endif
1529 max_length -= CHOMP_BUFFER;
1530 if (max_length > 0 && outlen > max_length)
1531 {
1532 logprintf (LOG_NOTQUIET, "The destination name is too long (%d), reducing to %d\n", outlen, max_length);
1533
1534 outlen = max_length;
1535 }
1536 GROW (dest, outlen);
1537
1538 // This should not happen, but it's impossible to argue with static analysis that it can't happen
1539 // (in theory it can). So give static analyzers a hint.
1540 if (!dest->base)
1541 return;
1542
1543 if (!quoted)
1544 {
1545 /* If there's nothing to quote, we can simply append the string
1546 without processing it again. */
1547 memcpy (TAIL (dest), b, outlen);
1548 }
1549 else
1550 {
1551 char *q = TAIL (dest);
1552 int i;
1553
1554 for (i = 0, p = b; p < e; p++)
1555 {
1556 if (!FILE_CHAR_TEST (*p, mask))
1557 {
1558 if (i == outlen)
1559 break;
1560 *q++ = *p;
1561 i++;
1562 }
1563 else if (i + 3 > outlen)
1564 break;
1565 else
1566 {
1567 unsigned char ch = *p;
1568 *q++ = '%';
1569 *q++ = XNUM_TO_DIGIT (ch >> 4);
1570 *q++ = XNUM_TO_DIGIT (ch & 0xf);
1571 i += 3;
1572 }
1573 }
1574 assert (q - TAIL (dest) <= outlen);
1575 }
1576
1577 /* Perform inline case transformation if required. */
1578 if (opt.restrict_files_case == restrict_lowercase
1579 || opt.restrict_files_case == restrict_uppercase)
1580 {
1581 char *q;
1582 for (q = TAIL (dest); q < TAIL (dest) + outlen; ++q)
1583 {
1584 if (opt.restrict_files_case == restrict_lowercase)
1585 *q = c_tolower (*q);
1586 else
1587 *q = c_toupper (*q);
1588 }
1589 }
1590
1591 TAIL_INCR (dest, outlen);
1592 append_null (dest);
1593
1594 if (unescaped && unescaped != buf)
1595 free (unescaped);
1596 }
1597
1598 #ifdef HAVE_ICONV
1599 static char *
1600 convert_fname (char *fname)
1601 {
1602 char *converted_fname;
1603 const char *from_encoding = opt.encoding_remote;
1604 const char *to_encoding = opt.locale;
1605 iconv_t cd;
1606 size_t len, done, inlen, outlen;
1607 char *s;
1608 const char *orig_fname;
1609
1610 /* Defaults for remote and local encodings. */
1611 if (!from_encoding)
1612 from_encoding = "UTF-8";
1613 if (!to_encoding)
1614 to_encoding = nl_langinfo (CODESET);
1615
1616 cd = iconv_open (to_encoding, from_encoding);
1617 if (cd == (iconv_t) (-1))
1618 {
1619 logprintf (LOG_VERBOSE, _ ("Conversion from %s to %s isn't supported\n"),
1620 quote_n (0, from_encoding), quote_n (1, to_encoding));
1621 return fname;
1622 }
1623
1624 orig_fname = fname;
1625 inlen = strlen (fname);
1626 len = outlen = inlen * 2;
1627 converted_fname = s = xmalloc (outlen + 1);
1628 done = 0;
1629
1630 for (;;)
1631 {
1632 errno = 0;
1633 if (iconv (cd, (ICONV_CONST char **) &fname, &inlen, &s, &outlen) == 0
1634 && iconv (cd, NULL, NULL, &s, &outlen) == 0)
1635 {
1636 *(converted_fname + len - outlen - done) = '\0';
1637 iconv_close (cd);
1638 DEBUGP (("Converted file name '%s' (%s) -> '%s' (%s)\n",
1639 orig_fname, from_encoding, converted_fname, to_encoding));
1640 xfree (orig_fname);
1641 return converted_fname;
1642 }
1643
1644 /* Incomplete or invalid multibyte sequence */
1645 if (errno == EINVAL || errno == EILSEQ || errno == 0)
1646 {
1647 if (errno)
1648 logprintf (LOG_VERBOSE,
1649 _ ("Incomplete or invalid multibyte sequence encountered\n"));
1650 else
1651 logprintf (LOG_VERBOSE,
1652 _ ("Unconvertable multibyte sequence encountered\n"));
1653 xfree (converted_fname);
1654 converted_fname = (char *) orig_fname;
1655 break;
1656 }
1657 else if (errno == E2BIG) /* Output buffer full */
1658 {
1659 done = len;
1660 len = outlen = done + inlen * 2;
1661 converted_fname = xrealloc (converted_fname, outlen + 1);
1662 s = converted_fname + done;
1663 }
1664 else /* Weird, we got an unspecified error */
1665 {
1666 logprintf (LOG_VERBOSE, _ ("Unhandled errno %d\n"), errno);
1667 xfree (converted_fname);
1668 converted_fname = (char *) orig_fname;
1669 break;
1670 }
1671 }
1672 DEBUGP (("Failed to convert file name '%s' (%s) -> '?' (%s)\n",
1673 orig_fname, from_encoding, to_encoding));
1674
1675 iconv_close (cd);
1676
1677 return converted_fname;
1678 }
1679 #else
1680 static char *
1681 convert_fname (char *fname)
1682 {
1683 return fname;
1684 }
1685 #endif
1686
1687 /* Append to DEST the directory structure that corresponds the
1688 directory part of URL's path. For example, if the URL is
1689 http://server/dir1/dir2/file, this appends "/dir1/dir2".
1690
1691 Each path element ("dir1" and "dir2" in the above example) is
1692 examined, url-unescaped, and re-escaped as file name element.
1693
1694 Additionally, it cuts as many directories from the path as
1695 specified by opt.cut_dirs. For example, if opt.cut_dirs is 1, it
1696 will produce "bar" for the above example. For 2 or more, it will
1697 produce "".
1698
1699 Each component of the path is quoted for use as file name. */
1700
1701 static void
1702 append_dir_structure (const struct url *u, struct growable *dest)
1703 {
1704 char *pathel, *next;
1705 int cut = opt.cut_dirs;
1706
1707 /* Go through the path components, de-URL-quote them, and quote them
1708 (if necessary) as file names. */
1709
1710 pathel = u->path;
1711 for (; (next = strchr (pathel, '/')) != NULL; pathel = next + 1)
1712 {
1713 if (cut-- > 0)
1714 continue;
1715 if (pathel == next)
1716 /* Ignore empty pathels. */
1717 continue;
1718
1719 if (dest->tail)
1720 append_char ('/', dest);
1721
1722 append_uri_pathel (pathel, next, true, dest);
1723 }
1724 }
1725
1726 /* Return a unique file name that matches the given URL as well as
1727 possible. Does not create directories on the file system. */
1728
1729 char *
1730 url_file_name (const struct url *u, char *replaced_filename)
1731 {
1732 struct growable fnres; /* stands for "file name result" */
1733 struct growable temp_fnres;
1734
1735 const char *u_file;
1736 char *fname, *unique, *fname_len_check;
1737 const char *index_filename = "index.html"; /* The default index file is index.html */
1738
1739 fnres.base = NULL;
1740 fnres.size = 0;
1741 fnres.tail = 0;
1742
1743 temp_fnres.base = NULL;
1744 temp_fnres.size = 0;
1745 temp_fnres.tail = 0;
1746
1747 /* If an alternative index file was defined, change index_filename */
1748 if (opt.default_page)
1749 index_filename = opt.default_page;
1750
1751
1752 /* Start with the directory prefix, if specified. */
1753 if (opt.dir_prefix)
1754 append_string (opt.dir_prefix, &fnres);
1755
1756 /* If "dirstruct" is turned on (typically the case with -r), add
1757 the host and port (unless those have been turned off) and
1758 directory structure. */
1759 /* All safe remote chars are unescaped and stored in temp_fnres,
1760 then converted to local and appended to fnres.
1761 Internationalized URL/IDN will produce punycode to lookup IP from DNS:
1762 https://en.wikipedia.org/wiki/URL
1763 https://en.wikipedia.org/wiki/Internationalized_domain_name
1764 Non-ASCII code chars in the path:
1765 https://en.wikipedia.org/wiki/List_of_Unicode_characters
1766 https://en.wikipedia.org/wiki/List_of_writing_systems */
1767 if (opt.dirstruct)
1768 {
1769 if (opt.protocol_directories)
1770 {
1771 if (temp_fnres.tail)
1772 append_char ('/', &temp_fnres);
1773 append_string (supported_schemes[u->scheme].name, &temp_fnres);
1774 }
1775 if (opt.add_hostdir)
1776 {
1777 if (temp_fnres.tail)
1778 append_char ('/', &temp_fnres);
1779 if (0 != strcmp (u->host, ".."))
1780 append_string (u->host, &temp_fnres);
1781 else
1782 /* Host name can come from the network; malicious DNS may
1783 allow ".." to be resolved, causing us to write to
1784 "../<file>". Defang such host names. */
1785 append_string ("%2E%2E", &temp_fnres);
1786 if (u->port != scheme_default_port (u->scheme))
1787 {
1788 char portstr[24];
1789 number_to_string (portstr, u->port);
1790 append_char (FN_PORT_SEP, &temp_fnres);
1791 append_string (portstr, &temp_fnres);
1792 }
1793 }
1794
1795 append_dir_structure (u, &temp_fnres);
1796 }
1797
1798 if (!replaced_filename)
1799 {
1800 /* Create the filename. */
1801 u_file = *u->file ? u->file : index_filename;
1802
1803 /* Append "?query" to the file name, even if empty,
1804 * and create fname_len_check. */
1805 if (u->query)
1806 fname_len_check = concat_strings (u_file, FN_QUERY_SEP_STR, u->query, NULL);
1807 else
1808 fname_len_check = strdupdelim (u_file, u_file + strlen (u_file));
1809 }
1810 else
1811 {
1812 u_file = replaced_filename;
1813 fname_len_check = strdupdelim (u_file, u_file + strlen (u_file));
1814 }
1815
1816 if (temp_fnres.tail)
1817 append_char ('/', &temp_fnres);
1818
1819 append_uri_pathel (fname_len_check,
1820 fname_len_check + strlen (fname_len_check), true, &temp_fnres);
1821
1822 /* Zero-terminate the temporary file name. */
1823 append_char ('\0', &temp_fnres);
1824
1825 /* convert all remote chars before length check and appending to local path */
1826 fname = convert_fname (temp_fnres.base);
1827 temp_fnres.base = NULL;
1828 temp_fnres.size = 0;
1829 temp_fnres.tail = 0;
1830 append_string (fname, &temp_fnres);
1831
1832 xfree (fname);
1833 xfree (fname_len_check);
1834
1835 /* The filename has already been 'cleaned' by append_uri_pathel() above. So,
1836 * just append it. */
1837 if (fnres.tail)
1838 append_char ('/', &fnres);
1839 append_string (temp_fnres.base, &fnres);
1840
1841 fname = fnres.base;
1842
1843 /* Make a final check that the path length is acceptable? */
1844 /* TODO: check fnres.base for path length problem */
1845
1846 xfree (temp_fnres.base);
1847
1848 /* Check the cases in which the unique extensions are not used:
1849 1) Clobbering is turned off (-nc).
1850 2) Retrieval with regetting.
1851 3) Timestamping is used.
1852 4) Hierarchy is built.
1853 5) Backups are specified.
1854
1855 The exception is the case when file does exist and is a
1856 directory (see `mkalldirs' for explanation). */
1857
1858 if (ALLOW_CLOBBER
1859 && !(file_exists_p (fname, NULL) && !file_non_directory_p (fname)))
1860 {
1861 unique = fname;
1862 }
1863 else
1864 {
1865 unique = unique_name_passthrough (fname);
1866 if (unique != fname)
1867 xfree (fname);
1868 }
1869
1870 /* On VMS, alter the name as required. */
1871 #ifdef __VMS
1872 {
1873 char *unique2;
1874
1875 unique2 = ods_conform( unique);
1876 if (unique2 != unique)
1877 {
1878 xfree (unique);
1879 unique = unique2;
1880 }
1881 }
1882 #endif /* def __VMS */
1883
1884 return unique;
1885 }
1886
1887 /* Resolve "." and ".." elements of PATH by destructively modifying
1888 PATH and return true if PATH has been modified, false otherwise.
1889
1890 The algorithm is in spirit similar to the one described in rfc1808,
1891 although implemented differently, in one pass. To recap, path
1892 elements containing only "." are removed, and ".." is taken to mean
1893 "back up one element". Single leading and trailing slashes are
1894 preserved.
1895
1896 For example, "a/b/c/./../d/.." will yield "a/b/". More exhaustive
1897 test examples are provided below. If you change anything in this
1898 function, run test_path_simplify to make sure you haven't broken a
1899 test case. */
1900
1901 static bool
1902 path_simplify (enum url_scheme scheme, char *path)
1903 {
1904 char *h = path; /* hare */
1905 char *t = path; /* tortoise */
1906 char *beg = path;
1907 char *end = strchr (path, '\0');
1908
1909 while (h < end)
1910 {
1911 /* Hare should be at the beginning of a path element. */
1912
1913 if (h[0] == '.' && (h[1] == '/' || h[1] == '\0'))
1914 {
1915 /* Ignore "./". */
1916 h += 2;
1917 }
1918 else if (h[0] == '.' && h[1] == '.' && (h[2] == '/' || h[2] == '\0'))
1919 {
1920 /* Handle "../" by retreating the tortoise by one path
1921 element -- but not past beginning. */
1922 if (t > beg)
1923 {
1924 /* Move backwards until T hits the beginning of the
1925 previous path element or the beginning of path. */
1926 for (--t; t > beg && t[-1] != '/'; t--)
1927 ;
1928 }
1929 else if (scheme == SCHEME_FTP
1930 #ifdef HAVE_SSL
1931 || scheme == SCHEME_FTPS
1932 #endif
1933 )
1934 {
1935 /* If we're at the beginning, copy the "../" literally
1936 and move the beginning so a later ".." doesn't remove
1937 it. This violates RFC 3986; but we do it for FTP
1938 anyway because there is otherwise no way to get at a
1939 parent directory, when the FTP server drops us in a
1940 non-root directory (which is not uncommon). */
1941 beg = t + 3;
1942 goto regular;
1943 }
1944 h += 3;
1945 }
1946 else
1947 {
1948 regular:
1949 /* A regular path element. If H hasn't advanced past T,
1950 simply skip to the next path element. Otherwise, copy
1951 the path element until the next slash. */
1952 if (t == h)
1953 {
1954 /* Skip the path element, including the slash. */
1955 while (h < end && *h != '/')
1956 t++, h++;
1957 if (h < end)
1958 t++, h++;
1959 }
1960 else
1961 {
1962 /* Copy the path element, including the final slash. */
1963 while (h < end && *h != '/')
1964 *t++ = *h++;
1965 if (h < end)
1966 *t++ = *h++;
1967 }
1968 }
1969 }
1970
1971 if (t != h)
1972 *t = '\0';
1973
1974 return t != h;
1975 }
1976
1977 /* Return the length of URL's path. Path is considered to be
1978 terminated by one or more of the ?query or ;params or #fragment,
1979 depending on the scheme. */
1980
1981 static const char *
1982 path_end (const char *url)
1983 {
1984 enum url_scheme scheme = url_scheme (url);
1985 const char *seps;
1986 if (scheme == SCHEME_INVALID)
1987 scheme = SCHEME_HTTP; /* use http semantics for rel links */
1988 /* +2 to ignore the first two separators ':' and '/' */
1989 seps = init_seps (scheme) + 2;
1990 return strpbrk_or_eos (url, seps);
1991 }
1992
1993 /* Find the last occurrence of character C in the range [b, e), or
1994 NULL, if none are present. */
1995 #define find_last_char(b, e, c) memrchr ((b), (c), (e) - (b))
1996
1997 /* Merge BASE with LINK and return the resulting URI.
1998
1999 Either of the URIs may be absolute or relative, complete with the
2000 host name, or path only. This tries to reasonably handle all
2001 foreseeable cases. It only employs minimal URL parsing, without
2002 knowledge of the specifics of schemes.
2003
2004 I briefly considered making this function call path_simplify after
2005 the merging process, as rfc1738 seems to suggest. This is a bad
2006 idea for several reasons: 1) it complexifies the code, and 2)
2007 url_parse has to simplify path anyway, so it's wasteful to boot. */
2008
2009 char *
2010 uri_merge (const char *base, const char *link)
2011 {
2012 int linklength;
2013 const char *end;
2014 char *merge;
2015
2016 if (url_has_scheme (link))
2017 return xstrdup (link);
2018
2019 /* We may not examine BASE past END. */
2020 end = path_end (base);
2021 linklength = strlen (link);
2022
2023 if (!*link)
2024 {
2025 /* Empty LINK points back to BASE, query string and all. */
2026 return xstrdup (base);
2027 }
2028 else if (*link == '?')
2029 {
2030 /* LINK points to the same location, but changes the query
2031 string. Examples: */
2032 /* uri_merge("path", "?new") -> "path?new" */
2033 /* uri_merge("path?foo", "?new") -> "path?new" */
2034 /* uri_merge("path?foo#bar", "?new") -> "path?new" */
2035 /* uri_merge("path#foo", "?new") -> "path?new" */
2036 int baselength = end - base;
2037 merge = xmalloc (baselength + linklength + 1);
2038 memcpy (merge, base, baselength);
2039 memcpy (merge + baselength, link, linklength);
2040 merge[baselength + linklength] = '\0';
2041 }
2042 else if (*link == '#')
2043 {
2044 /* uri_merge("path", "#new") -> "path#new" */
2045 /* uri_merge("path#foo", "#new") -> "path#new" */
2046 /* uri_merge("path?foo", "#new") -> "path?foo#new" */
2047 /* uri_merge("path?foo#bar", "#new") -> "path?foo#new" */
2048 int baselength;
2049 const char *end1 = strchr (base, '#');
2050 if (!end1)
2051 end1 = base + strlen (base);
2052 baselength = end1 - base;
2053 merge = xmalloc (baselength + linklength + 1);
2054 memcpy (merge, base, baselength);
2055 memcpy (merge + baselength, link, linklength);
2056 merge[baselength + linklength] = '\0';
2057 }
2058 else if (*link == '/' && *(link + 1) == '/')
2059 {
2060 /* LINK begins with "//" and so is a net path: we need to
2061 replace everything after (and including) the double slash
2062 with LINK. */
2063
2064 /* uri_merge("foo", "//new/bar") -> "//new/bar" */
2065 /* uri_merge("//old/foo", "//new/bar") -> "//new/bar" */
2066 /* uri_merge("http://old/foo", "//new/bar") -> "http://new/bar" */
2067
2068 int span;
2069 const char *slash;
2070 const char *start_insert;
2071
2072 /* Look for first slash. */
2073 slash = memchr (base, '/', end - base);
2074 /* If found slash and it is a double slash, then replace
2075 from this point, else default to replacing from the
2076 beginning. */
2077 if (slash && *(slash + 1) == '/')
2078 start_insert = slash;
2079 else
2080 start_insert = base;
2081
2082 span = start_insert - base;
2083 merge = xmalloc (span + linklength + 1);
2084 if (span)
2085 memcpy (merge, base, span);
2086 memcpy (merge + span, link, linklength);
2087 merge[span + linklength] = '\0';
2088 }
2089 else if (*link == '/')
2090 {
2091 /* LINK is an absolute path: we need to replace everything
2092 after (and including) the FIRST slash with LINK.
2093
2094 So, if BASE is "http://host/whatever/foo/bar", and LINK is
2095 "/qux/xyzzy", our result should be
2096 "http://host/qux/xyzzy". */
2097 int span;
2098 const char *slash;
2099 const char *start_insert = NULL; /* for gcc to shut up. */
2100 const char *pos = base;
2101 bool seen_slash_slash = false;
2102 /* We're looking for the first slash, but want to ignore
2103 double slash. */
2104 again:
2105 slash = memchr (pos, '/', end - pos);
2106 if (slash && !seen_slash_slash)
2107 if (*(slash + 1) == '/')
2108 {
2109 pos = slash + 2;
2110 seen_slash_slash = true;
2111 goto again;
2112 }
2113
2114 /* At this point, SLASH is the location of the first / after
2115 "//", or the first slash altogether. START_INSERT is the
2116 pointer to the location where LINK will be inserted. When
2117 examining the last two examples, keep in mind that LINK
2118 begins with '/'. */
2119
2120 if (!slash && !seen_slash_slash)
2121 /* example: "foo" */
2122 /* ^ */
2123 start_insert = base;
2124 else if (!slash && seen_slash_slash)
2125 /* example: "http://foo" */
2126 /* ^ */
2127 start_insert = end;
2128 else if (slash && !seen_slash_slash)
2129 /* example: "foo/bar" */
2130 /* ^ */
2131 start_insert = base;
2132 else if (slash && seen_slash_slash)
2133 /* example: "http://something/" */
2134 /* ^ */
2135 start_insert = slash;
2136
2137 span = start_insert - base;
2138 merge = xmalloc (span + linklength + 1);
2139 if (span)
2140 memcpy (merge, base, span);
2141 memcpy (merge + span, link, linklength);
2142 merge[span + linklength] = '\0';
2143 }
2144 else
2145 {
2146 /* LINK is a relative URL: we need to replace everything
2147 after last slash (possibly empty) with LINK.
2148
2149 So, if BASE is "whatever/foo/bar", and LINK is "qux/xyzzy",
2150 our result should be "whatever/foo/qux/xyzzy". */
2151 bool need_explicit_slash = false;
2152 int span;
2153 const char *start_insert;
2154 const char *last_slash = find_last_char (base, end, '/');
2155 if (!last_slash)
2156 {
2157 /* No slash found at all. Replace what we have with LINK. */
2158 start_insert = base;
2159 }
2160 else if (last_slash && last_slash >= base + 2
2161 && last_slash[-2] == ':' && last_slash[-1] == '/')
2162 {
2163 /* example: http://host" */
2164 /* ^ */
2165 start_insert = end + 1;
2166 need_explicit_slash = true;
2167 }
2168 else
2169 {
2170 /* example: "whatever/foo/bar" */
2171 /* ^ */
2172 start_insert = last_slash + 1;
2173 }
2174
2175 span = start_insert - base;
2176 merge = xmalloc (span + linklength + 1);
2177 if (span)
2178 memcpy (merge, base, span);
2179 if (need_explicit_slash)
2180 merge[span - 1] = '/';
2181 memcpy (merge + span, link, linklength);
2182 merge[span + linklength] = '\0';
2183 }
2184
2185 return merge;
2186 }
2187
2188 #define APPEND(p, s) do { \
2189 int len = strlen (s); \
2190 memcpy (p, s, len); \
2191 p += len; \
2192 } while (0)
2193
2194 /* Use this instead of password when the actual password is supposed
2195 to be hidden. We intentionally use a generic string without giving
2196 away the number of characters in the password, like previous
2197 versions did. */
2198 #define HIDDEN_PASSWORD "*password*"
2199
2200 /* Recreate the URL string from the data in URL.
2201
2202 If HIDE is true (as it is when we're calling this on a URL we plan
2203 to print, but not when calling it to canonicalize a URL for use
2204 within the program), password will be hidden. Unsafe characters in
2205 the URL will be quoted. */
2206
2207 char *
2208 url_string (const struct url *url, enum url_auth_mode auth_mode)
2209 {
2210 int size;
2211 char *result, *p;
2212 char *quoted_host, *quoted_user = NULL, *quoted_passwd = NULL;
2213
2214 int scheme_port = supported_schemes[url->scheme].default_port;
2215 const char *scheme_str = supported_schemes[url->scheme].leading_string;
2216 int fplen = full_path_length (url);
2217
2218 bool brackets_around_host;
2219
2220 assert (scheme_str != NULL);
2221
2222 /* Make sure the user name and password are quoted. */
2223 if (url->user)
2224 {
2225 if (auth_mode != URL_AUTH_HIDE)
2226 {
2227 quoted_user = url_escape_allow_passthrough (url->user);
2228 if (url->passwd)
2229 {
2230 if (auth_mode == URL_AUTH_HIDE_PASSWD)
2231 quoted_passwd = (char *) HIDDEN_PASSWORD;
2232 else
2233 quoted_passwd = url_escape_allow_passthrough (url->passwd);
2234 }
2235 }
2236 }
2237
2238 /* In the unlikely event that the host name contains non-printable
2239 characters, quote it for displaying to the user. */
2240 quoted_host = url_escape_allow_passthrough (url->host);
2241
2242 /* Undo the quoting of colons that URL escaping performs. IPv6
2243 addresses may legally contain colons, and in that case must be
2244 placed in square brackets. */
2245 if (quoted_host != url->host)
2246 unescape_single_char (quoted_host, ':');
2247 brackets_around_host = strchr (quoted_host, ':') != NULL;
2248
2249 size = (strlen (scheme_str)
2250 + strlen (quoted_host)
2251 + (brackets_around_host ? 2 : 0)
2252 + fplen
2253 + 1);
2254 if (url->port != scheme_port)
2255 size += 1 + numdigit (url->port);
2256 if (quoted_user)
2257 {
2258 size += 1 + strlen (quoted_user);
2259 if (quoted_passwd)
2260 size += 1 + strlen (quoted_passwd);
2261 }
2262
2263 p = result = xmalloc (size);
2264
2265 APPEND (p, scheme_str);
2266 if (quoted_user)
2267 {
2268 APPEND (p, quoted_user);
2269 if (quoted_passwd)
2270 {
2271 *p++ = ':';
2272 APPEND (p, quoted_passwd);
2273 }
2274 *p++ = '@';
2275 }
2276
2277 if (brackets_around_host)
2278 *p++ = '[';
2279 APPEND (p, quoted_host);
2280 if (brackets_around_host)
2281 *p++ = ']';
2282 if (url->port != scheme_port)
2283 {
2284 *p++ = ':';
2285 p = number_to_string (p, url->port);
2286 }
2287
2288 full_path_write (url, p);
2289 p += fplen;
2290 *p++ = '\0';
2291
2292 assert (p - result == size);
2293
2294 if (quoted_user && quoted_user != url->user)
2295 xfree (quoted_user);
2296 if (quoted_passwd && auth_mode == URL_AUTH_SHOW
2297 && quoted_passwd != url->passwd)
2298 xfree (quoted_passwd);
2299 if (quoted_host != url->host)
2300 xfree (quoted_host);
2301
2302 return result;
2303 }
2304
2305 /* Return true if scheme a is similar to scheme b.
2306
2307 Schemes are similar if they are equal. If SSL is supported, schemes
2308 are also similar if one is http (SCHEME_HTTP) and the other is https
2309 (SCHEME_HTTPS). */
2310 bool
2311 schemes_are_similar_p (enum url_scheme a, enum url_scheme b)
2312 {
2313 if (a == b)
2314 return true;
2315 #ifdef HAVE_SSL
2316 if ((a == SCHEME_HTTP && b == SCHEME_HTTPS)
2317 || (a == SCHEME_HTTPS && b == SCHEME_HTTP))
2318 return true;
2319 #endif
2320 return false;
2321 }
2322
2323 static int
2324 getchar_from_escaped_string (const char *str, char *c)
2325 {
2326 const char *p = str;
2327
2328 assert (str && *str);
2329 assert (c);
2330
2331 if (p[0] == '%')
2332 {
2333 if (!c_isxdigit(p[1]) || !c_isxdigit(p[2]))
2334 {
2335 *c = '%';
2336 return 1;
2337 }
2338 else
2339 {
2340 if (p[2] == 0)
2341 return 0; /* error: invalid string */
2342
2343 *c = X2DIGITS_TO_NUM (p[1], p[2]);
2344 if (URL_RESERVED_CHAR(*c))
2345 {
2346 *c = '%';
2347 return 1;
2348 }
2349 else
2350 return 3;
2351 }
2352 }
2353 else
2354 {
2355 *c = p[0];
2356 }
2357
2358 return 1;
2359 }
2360
2361 bool
2362 are_urls_equal (const char *u1, const char *u2)
2363 {
2364 const char *p, *q;
2365 int pp, qq;
2366 char ch1, ch2;
2367 assert(u1 && u2);
2368
2369 p = u1;
2370 q = u2;
2371
2372 while (*p && *q
2373 && (pp = getchar_from_escaped_string (p, &ch1))
2374 && (qq = getchar_from_escaped_string (q, &ch2))
2375 && (c_tolower(ch1) == c_tolower(ch2)))
2376 {
2377 p += pp;
2378 q += qq;
2379 }
2380
2381 return (*p == 0 && *q == 0 ? true : false);
2382 }
2383
2384 #ifdef TESTING
2385 /* Debugging and testing support for path_simplify. */
2386
2387 #if 0
2388 /* Debug: run path_simplify on PATH and return the result in a new
2389 string. Useful for calling from the debugger. */
2390 static char *
2391 ps (char *path)
2392 {
2393 char *copy = xstrdup (path);
2394 path_simplify (copy);
2395 return copy;
2396 }
2397 #endif
2398
2399 static const char *
2400 run_test (const char *test, const char *expected_result, enum url_scheme scheme,
2401 bool expected_change)
2402 {
2403 char *test_copy = xstrdup (test);
2404 bool modified = path_simplify (scheme, test_copy);
2405
2406 if (0 != strcmp (test_copy, expected_result))
2407 {
2408 printf ("Failed path_simplify(\"%s\"): expected \"%s\", got \"%s\".\n",
2409 test, expected_result, test_copy);
2410 mu_assert ("", 0);
2411 }
2412 if (modified != expected_change)
2413 {
2414 if (expected_change)
2415 printf ("Expected modification with path_simplify(\"%s\").\n",
2416 test);
2417 else
2418 printf ("Expected no modification with path_simplify(\"%s\").\n",
2419 test);
2420 }
2421 xfree (test_copy);
2422 mu_assert ("", modified == expected_change);
2423 return NULL;
2424 }
2425
2426 const char *
2427 test_path_simplify (void)
2428 {
2429 static const struct {
2430 const char *test, *result;
2431 enum url_scheme scheme;
2432 bool should_modify;
2433 } tests[] = {
2434 { "", "", SCHEME_HTTP, false },
2435 { ".", "", SCHEME_HTTP, true },
2436 { "./", "", SCHEME_HTTP, true },
2437 { "..", "", SCHEME_HTTP, true },
2438 { "../", "", SCHEME_HTTP, true },
2439 { "..", "..", SCHEME_FTP, false },
2440 { "../", "../", SCHEME_FTP, false },
2441 { "foo", "foo", SCHEME_HTTP, false },
2442 { "foo/bar", "foo/bar", SCHEME_HTTP, false },
2443 { "foo///bar", "foo///bar", SCHEME_HTTP, false },
2444 { "foo/.", "foo/", SCHEME_HTTP, true },
2445 { "foo/./", "foo/", SCHEME_HTTP, true },
2446 { "foo./", "foo./", SCHEME_HTTP, false },
2447 { "foo/../bar", "bar", SCHEME_HTTP, true },
2448 { "foo/../bar/", "bar/", SCHEME_HTTP, true },
2449 { "foo/bar/..", "foo/", SCHEME_HTTP, true },
2450 { "foo/bar/../x", "foo/x", SCHEME_HTTP, true },
2451 { "foo/bar/../x/", "foo/x/", SCHEME_HTTP, true },
2452 { "foo/..", "", SCHEME_HTTP, true },
2453 { "foo/../..", "", SCHEME_HTTP, true },
2454 { "foo/../../..", "", SCHEME_HTTP, true },
2455 { "foo/../../bar/../../baz", "baz", SCHEME_HTTP, true },
2456 { "foo/../..", "..", SCHEME_FTP, true },
2457 { "foo/../../..", "../..", SCHEME_FTP, true },
2458 { "foo/../../bar/../../baz", "../../baz", SCHEME_FTP, true },
2459 { "a/b/../../c", "c", SCHEME_HTTP, true },
2460 { "./a/../b", "b", SCHEME_HTTP, true }
2461 };
2462 unsigned i;
2463
2464 for (i = 0; i < countof (tests); i++)
2465 {
2466 const char *message;
2467 const char *test = tests[i].test;
2468 const char *expected_result = tests[i].result;
2469 enum url_scheme scheme = tests[i].scheme;
2470 bool expected_change = tests[i].should_modify;
2471
2472 message = run_test (test, expected_result, scheme, expected_change);
2473 if (message) return message;
2474 }
2475 return NULL;
2476 }
2477
2478 const char *
2479 test_append_uri_pathel(void)
2480 {
2481 unsigned i;
2482 static const struct {
2483 const char *original_url;
2484 const char *input;
2485 bool escaped;
2486 const char *expected_result;
2487 } test_array[] = {
2488 { "http://www.yoyodyne.com/path/", "somepage.html", false, "http://www.yoyodyne.com/path/somepage.html" },
2489 };
2490
2491 for (i = 0; i < countof(test_array); ++i)
2492 {
2493 struct growable dest;
2494 const char *p = test_array[i].input;
2495
2496 memset (&dest, 0, sizeof (dest));
2497
2498 append_string (test_array[i].original_url, &dest);
2499 append_uri_pathel (p, p + strlen(p), test_array[i].escaped, &dest);
2500
2501 mu_assert ("test_append_uri_pathel: wrong result",
2502 strcmp (dest.base, test_array[i].expected_result) == 0);
2503 xfree (dest.base);
2504 }
2505
2506 return NULL;
2507 }
2508
2509 const char *
2510 test_are_urls_equal(void)
2511 {
2512 unsigned i;
2513 static const struct {
2514 const char *url1;
2515 const char *url2;
2516 bool expected_result;
2517 } test_array[] = {
2518 { "http://www.adomain.com/apath/", "http://www.adomain.com/apath/", true },
2519 { "http://www.adomain.com/apath/", "http://www.adomain.com/anotherpath/", false },
2520 { "http://www.adomain.com/apath/", "http://www.anotherdomain.com/path/", false },
2521 { "http://www.adomain.com/~path/", "http://www.adomain.com/%7epath/", true },
2522 { "http://www.adomain.com/longer-path/", "http://www.adomain.com/path/", false },
2523 { "http://www.adomain.com/path%2f", "http://www.adomain.com/path/", false },
2524 };
2525
2526 for (i = 0; i < countof(test_array); ++i)
2527 {
2528 mu_assert ("test_are_urls_equal: wrong result",
2529 are_urls_equal (test_array[i].url1, test_array[i].url2) == test_array[i].expected_result);
2530 }
2531
2532 return NULL;
2533 }
2534
2535 const char *
2536 test_uri_merge(void)
2537 {
2538 static const struct test_data {
2539 const char *url;
2540 const char *link;
2541 const char *expected;
2542 } test_data[] = {
2543 { "http://www.yoyodyne.com/path/", "somepage.html", "http://www.yoyodyne.com/path/somepage.html" },
2544 { "http://example.com/path/", "//other.com/somepage.html", "http://other.com/somepage.html" },
2545 { "https://example.com/path/", "//other.com/somepage.html", "https://other.com/somepage.html" },
2546 };
2547
2548 for (unsigned i = 0; i < countof(test_data); ++i)
2549 {
2550 const struct test_data *t = &test_data[i];
2551 char *result = uri_merge (t->url, t->link);
2552 bool ok = strcmp (result, t->expected) == 0;
2553 if (!ok)
2554 return aprintf ("test_uri_merge [%u]: expected '%s', got '%s'", i, t->expected, result);
2555
2556 xfree (result);
2557 }
2558
2559 return NULL;
2560 }
2561
2562 #endif /* TESTING */
2563
2564 /*
2565 * vim: et ts=2 sw=2
2566 */