Debian Codegraph @

deps/curl/src/tool_operate.c

raw
1 /***************************************************************************
2 * _ _ ____ _
3 * Project ___| | | | _ \| |
4 * / __| | | | |_) | |
5 * | (__| |_| | _ <| |___
6 * \___|\___/|_| \_\_____|
7 *
8 * Copyright (C) 1998 - 2020, Daniel Stenberg, <[email protected]>, et al.
9 *
10 * This software is licensed as described in the file COPYING, which
11 * you should have received as part of this distribution. The terms
12 * are also available at https://curl.haxx.se/docs/copyright.html.
13 *
14 * You may opt to use, copy, modify, merge, publish, distribute and/or sell
15 * copies of the Software, and permit persons to whom the Software is
16 * furnished to do so, under the terms of the COPYING file.
17 *
18 * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
19 * KIND, either express or implied.
20 *
21 ***************************************************************************/
22 #include "tool_setup.h"
23
24 #ifdef HAVE_FCNTL_H
25 # include <fcntl.h>
26 #endif
27
28 #ifdef HAVE_LOCALE_H
29 # include <locale.h>
30 #endif
31
32 #ifdef HAVE_SYS_SELECT_H
33 # include <sys/select.h>
34 #endif
35
36 #ifdef __VMS
37 # include <fabdef.h>
38 #endif
39
40 #ifdef __AMIGA__
41 # include <proto/dos.h>
42 #endif
43
44 #include "strcase.h"
45
46 #define ENABLE_CURLX_PRINTF
47 /* use our own printf() functions */
48 #include "curlx.h"
49
50 #include "tool_binmode.h"
51 #include "tool_cfgable.h"
52 #include "tool_cb_dbg.h"
53 #include "tool_cb_hdr.h"
54 #include "tool_cb_prg.h"
55 #include "tool_cb_rea.h"
56 #include "tool_cb_see.h"
57 #include "tool_cb_wrt.h"
58 #include "tool_dirhie.h"
59 #include "tool_doswin.h"
60 #include "tool_easysrc.h"
61 #include "tool_filetime.h"
62 #include "tool_getparam.h"
63 #include "tool_helpers.h"
64 #include "tool_homedir.h"
65 #include "tool_libinfo.h"
66 #include "tool_main.h"
67 #include "tool_metalink.h"
68 #include "tool_msgs.h"
69 #include "tool_operate.h"
70 #include "tool_operhlp.h"
71 #include "tool_paramhlp.h"
72 #include "tool_parsecfg.h"
73 #include "tool_setopt.h"
74 #include "tool_sleep.h"
75 #include "tool_urlglob.h"
76 #include "tool_util.h"
77 #include "tool_writeout.h"
78 #include "tool_xattr.h"
79 #include "tool_vms.h"
80 #include "tool_help.h"
81 #include "tool_hugehelp.h"
82 #include "tool_progress.h"
83
84 #include "memdebug.h" /* keep this as LAST include */
85
86 #ifdef CURLDEBUG
87 /* libcurl's debug builds provide an extra function */
88 CURLcode curl_easy_perform_ev(CURL *easy);
89 #endif
90
91 #define CURLseparator "--_curl_--"
92
93 #ifndef O_BINARY
94 /* since O_BINARY as used in bitmasks, setting it to zero makes it usable in
95 source code but yet it doesn't ruin anything */
96 # define O_BINARY 0
97 #endif
98
99 #define CURL_CA_CERT_ERRORMSG \
100 "More details here: https://curl.haxx.se/docs/sslcerts.html\n\n" \
101 "curl failed to verify the legitimacy of the server and therefore " \
102 "could not\nestablish a secure connection to it. To learn more about " \
103 "this situation and\nhow to fix it, please visit the web page mentioned " \
104 "above.\n"
105
106 static CURLcode single_transfer(struct GlobalConfig *global,
107 struct OperationConfig *config,
108 CURLSH *share,
109 bool capath_from_env,
110 bool *added);
111 static CURLcode create_transfer(struct GlobalConfig *global,
112 CURLSH *share,
113 bool *added);
114
115 static bool is_fatal_error(CURLcode code)
116 {
117 switch(code) {
118 case CURLE_FAILED_INIT:
119 case CURLE_OUT_OF_MEMORY:
120 case CURLE_UNKNOWN_OPTION:
121 case CURLE_FUNCTION_NOT_FOUND:
122 case CURLE_BAD_FUNCTION_ARGUMENT:
123 /* critical error */
124 return TRUE;
125 default:
126 break;
127 }
128
129 /* no error or not critical */
130 return FALSE;
131 }
132
133 /*
134 * Check if a given string is a PKCS#11 URI
135 */
136 static bool is_pkcs11_uri(const char *string)
137 {
138 if(curl_strnequal(string, "pkcs11:", 7)) {
139 return TRUE;
140 }
141 else {
142 return FALSE;
143 }
144 }
145
146 #ifdef __VMS
147 /*
148 * get_vms_file_size does what it takes to get the real size of the file
149 *
150 * For fixed files, find out the size of the EOF block and adjust.
151 *
152 * For all others, have to read the entire file in, discarding the contents.
153 * Most posted text files will be small, and binary files like zlib archives
154 * and CD/DVD images should be either a STREAM_LF format or a fixed format.
155 *
156 */
157 static curl_off_t vms_realfilesize(const char *name,
158 const struct_stat *stat_buf)
159 {
160 char buffer[8192];
161 curl_off_t count;
162 int ret_stat;
163 FILE * file;
164
165 /* !checksrc! disable FOPENMODE 1 */
166 file = fopen(name, "r"); /* VMS */
167 if(file == NULL) {
168 return 0;
169 }
170 count = 0;
171 ret_stat = 1;
172 while(ret_stat > 0) {
173 ret_stat = fread(buffer, 1, sizeof(buffer), file);
174 if(ret_stat != 0)
175 count += ret_stat;
176 }
177 fclose(file);
178
179 return count;
180 }
181
182 /*
183 *
184 * VmsSpecialSize checks to see if the stat st_size can be trusted and
185 * if not to call a routine to get the correct size.
186 *
187 */
188 static curl_off_t VmsSpecialSize(const char *name,
189 const struct_stat *stat_buf)
190 {
191 switch(stat_buf->st_fab_rfm) {
192 case FAB$C_VAR:
193 case FAB$C_VFC:
194 return vms_realfilesize(name, stat_buf);
195 break;
196 default:
197 return stat_buf->st_size;
198 }
199 }
200 #endif /* __VMS */
201
202 #define BUFFER_SIZE (100*1024)
203
204 struct per_transfer *transfers; /* first node */
205 static struct per_transfer *transfersl; /* last node */
206
207 /* add_per_transfer creates a new 'per_transfer' node in the linked
208 list of transfers */
209 static CURLcode add_per_transfer(struct per_transfer **per)
210 {
211 struct per_transfer *p;
212 p = calloc(sizeof(struct per_transfer), 1);
213 if(!p)
214 return CURLE_OUT_OF_MEMORY;
215 if(!transfers)
216 /* first entry */
217 transfersl = transfers = p;
218 else {
219 /* make the last node point to the new node */
220 transfersl->next = p;
221 /* make the new node point back to the formerly last node */
222 p->prev = transfersl;
223 /* move the last node pointer to the new entry */
224 transfersl = p;
225 }
226 *per = p;
227 all_xfers++; /* count total number of transfers added */
228 return CURLE_OK;
229 }
230
231 /* Remove the specified transfer from the list (and free it), return the next
232 in line */
233 static struct per_transfer *del_per_transfer(struct per_transfer *per)
234 {
235 struct per_transfer *n;
236 struct per_transfer *p;
237 DEBUGASSERT(transfers);
238 DEBUGASSERT(transfersl);
239 DEBUGASSERT(per);
240
241 n = per->next;
242 p = per->prev;
243
244 if(p)
245 p->next = n;
246 else
247 transfers = n;
248
249 if(n)
250 n->prev = p;
251 else
252 transfersl = p;
253
254 free(per);
255
256 return n;
257 }
258
259 static CURLcode pre_transfer(struct GlobalConfig *global,
260 struct per_transfer *per)
261 {
262 curl_off_t uploadfilesize = -1;
263 struct_stat fileinfo;
264 CURLcode result = CURLE_OK;
265
266 if(per->separator_err)
267 fprintf(global->errors, "%s\n", per->separator_err);
268 if(per->separator)
269 printf("%s\n", per->separator);
270
271 if(per->uploadfile && !stdin_upload(per->uploadfile)) {
272 /* VMS Note:
273 *
274 * Reading binary from files can be a problem... Only FIXED, VAR
275 * etc WITHOUT implied CC will work Others need a \n appended to a
276 * line
277 *
278 * - Stat gives a size but this is UNRELIABLE in VMS As a f.e. a
279 * fixed file with implied CC needs to have a byte added for every
280 * record processed, this can by derived from Filesize & recordsize
281 * for VARiable record files the records need to be counted! for
282 * every record add 1 for linefeed and subtract 2 for the record
283 * header for VARIABLE header files only the bare record data needs
284 * to be considered with one appended if implied CC
285 */
286 #ifdef __VMS
287 /* Calculate the real upload size for VMS */
288 per->infd = -1;
289 if(stat(per->uploadfile, &fileinfo) == 0) {
290 fileinfo.st_size = VmsSpecialSize(uploadfile, &fileinfo);
291 switch(fileinfo.st_fab_rfm) {
292 case FAB$C_VAR:
293 case FAB$C_VFC:
294 case FAB$C_STMCR:
295 per->infd = open(per->uploadfile, O_RDONLY | O_BINARY);
296 break;
297 default:
298 per->infd = open(per->uploadfile, O_RDONLY | O_BINARY,
299 "rfm=stmlf", "ctx=stm");
300 }
301 }
302 if(per->infd == -1)
303 #else
304 per->infd = open(per->uploadfile, O_RDONLY | O_BINARY);
305 if((per->infd == -1) || fstat(per->infd, &fileinfo))
306 #endif
307 {
308 helpf(global->errors, "Can't open '%s'!\n", per->uploadfile);
309 if(per->infd != -1) {
310 close(per->infd);
311 per->infd = STDIN_FILENO;
312 }
313 return CURLE_READ_ERROR;
314 }
315 per->infdopen = TRUE;
316
317 /* we ignore file size for char/block devices, sockets, etc. */
318 if(S_ISREG(fileinfo.st_mode))
319 uploadfilesize = fileinfo.st_size;
320
321 if(uploadfilesize != -1)
322 my_setopt(per->curl, CURLOPT_INFILESIZE_LARGE, uploadfilesize);
323 per->input.fd = per->infd;
324 }
325 return result;
326 }
327
328 /*
329 * Call this after a transfer has completed.
330 */
331 static CURLcode post_per_transfer(struct GlobalConfig *global,
332 struct per_transfer *per,
333 CURLcode result,
334 bool *retryp)
335 {
336 struct OutStruct *outs = &per->outs;
337 CURL *curl = per->curl;
338 struct OperationConfig *config = per->config;
339
340 if(!curl || !config)
341 return result;
342
343 *retryp = FALSE;
344
345 if(per->infdopen)
346 close(per->infd);
347
348 #ifdef __VMS
349 if(is_vms_shell()) {
350 /* VMS DCL shell behavior */
351 if(!global->showerror)
352 vms_show = VMSSTS_HIDE;
353 }
354 else
355 #endif
356 if(config->synthetic_error) {
357 ;
358 }
359 else if(result && global->showerror) {
360 fprintf(global->errors, "curl: (%d) %s\n", result,
361 (per->errorbuffer[0]) ? per->errorbuffer :
362 curl_easy_strerror(result));
363 if(result == CURLE_PEER_FAILED_VERIFICATION)
364 fputs(CURL_CA_CERT_ERRORMSG, global->errors);
365 }
366
367 /* Set file extended attributes */
368 if(!result && config->xattr && outs->fopened && outs->stream) {
369 int rc = fwrite_xattr(curl, fileno(outs->stream));
370 if(rc)
371 warnf(config->global, "Error setting extended attributes: %s\n",
372 strerror(errno));
373 }
374
375 if(!result && !outs->stream && !outs->bytes) {
376 /* we have received no data despite the transfer was successful
377 ==> force cration of an empty output file (if an output file
378 was specified) */
379 long cond_unmet = 0L;
380 /* do not create (or even overwrite) the file in case we get no
381 data because of unmet condition */
382 curl_easy_getinfo(curl, CURLINFO_CONDITION_UNMET, &cond_unmet);
383 if(!cond_unmet && !tool_create_output_file(outs, config))
384 result = CURLE_WRITE_ERROR;
385 }
386
387 if(!outs->s_isreg && outs->stream) {
388 /* Dump standard stream buffered data */
389 int rc = fflush(outs->stream);
390 if(!result && rc) {
391 /* something went wrong in the writing process */
392 result = CURLE_WRITE_ERROR;
393 fprintf(global->errors, "(%d) Failed writing body\n", result);
394 }
395 }
396
397 #ifdef USE_METALINK
398 if(per->metalink && !per->metalink_next_res)
399 fprintf(global->errors, "Metalink: fetching (%s) from (%s) OK\n",
400 per->mlfile->filename, per->this_url);
401
402 if(!per->metalink && config->use_metalink && result == CURLE_OK) {
403 int rv = parse_metalink(config, outs, per->this_url);
404 if(!rv) {
405 fprintf(config->global->errors, "Metalink: parsing (%s) OK\n",
406 per->this_url);
407 }
408 else if(rv == -1)
409 fprintf(config->global->errors, "Metalink: parsing (%s) FAILED\n",
410 per->this_url);
411 }
412 else if(per->metalink && result == CURLE_OK && !per->metalink_next_res) {
413 int rv;
414 (void)fflush(outs->stream);
415 rv = metalink_check_hash(global, per->mlfile, outs->filename);
416 if(!rv)
417 per->metalink_next_res = 1;
418 }
419 #endif /* USE_METALINK */
420
421 #ifdef USE_METALINK
422 if(outs->metalink_parser)
423 metalink_parser_context_delete(outs->metalink_parser);
424 #endif /* USE_METALINK */
425
426 if(outs->is_cd_filename && outs->stream && !global->mute &&
427 outs->filename)
428 printf("curl: Saved to filename '%s'\n", outs->filename);
429
430 /* if retry-max-time is non-zero, make sure we haven't exceeded the
431 time */
432 if(per->retry_numretries &&
433 (!config->retry_maxtime ||
434 (tvdiff(tvnow(), per->retrystart) <
435 config->retry_maxtime*1000L)) ) {
436 enum {
437 RETRY_NO,
438 RETRY_TIMEOUT,
439 RETRY_CONNREFUSED,
440 RETRY_HTTP,
441 RETRY_FTP,
442 RETRY_LAST /* not used */
443 } retry = RETRY_NO;
444 long response;
445 if((CURLE_OPERATION_TIMEDOUT == result) ||
446 (CURLE_COULDNT_RESOLVE_HOST == result) ||
447 (CURLE_COULDNT_RESOLVE_PROXY == result) ||
448 (CURLE_FTP_ACCEPT_TIMEOUT == result))
449 /* retry timeout always */
450 retry = RETRY_TIMEOUT;
451 else if(config->retry_connrefused &&
452 (CURLE_COULDNT_CONNECT == result)) {
453 long oserrno;
454 curl_easy_getinfo(curl, CURLINFO_OS_ERRNO, &oserrno);
455 if(ECONNREFUSED == oserrno)
456 retry = RETRY_CONNREFUSED;
457 }
458 else if((CURLE_OK == result) ||
459 (config->failonerror &&
460 (CURLE_HTTP_RETURNED_ERROR == result))) {
461 /* If it returned OK. _or_ failonerror was enabled and it
462 returned due to such an error, check for HTTP transient
463 errors to retry on. */
464 long protocol;
465 curl_easy_getinfo(curl, CURLINFO_PROTOCOL, &protocol);
466 if((protocol == CURLPROTO_HTTP) || (protocol == CURLPROTO_HTTPS)) {
467 /* This was HTTP(S) */
468 curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &response);
469
470 switch(response) {
471 case 429: /* Too Many Requests (RFC6585) */
472 case 500: /* Internal Server Error */
473 case 502: /* Bad Gateway */
474 case 503: /* Service Unavailable */
475 case 504: /* Gateway Timeout */
476 retry = RETRY_HTTP;
477 /*
478 * At this point, we have already written data to the output
479 * file (or terminal). If we write to a file, we must rewind
480 * or close/re-open the file so that the next attempt starts
481 * over from the beginning.
482 *
483 * TODO: similar action for the upload case. We might need
484 * to start over reading from a previous point if we have
485 * uploaded something when this was returned.
486 */
487 break;
488 }
489 }
490 } /* if CURLE_OK */
491 else if(result) {
492 long protocol;
493
494 curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &response);
495 curl_easy_getinfo(curl, CURLINFO_PROTOCOL, &protocol);
496
497 if((protocol == CURLPROTO_FTP || protocol == CURLPROTO_FTPS) &&
498 response / 100 == 4)
499 /*
500 * This is typically when the FTP server only allows a certain
501 * amount of users and we are not one of them. All 4xx codes
502 * are transient.
503 */
504 retry = RETRY_FTP;
505 }
506
507 if(retry) {
508 long sleeptime = 0;
509 curl_off_t retry_after = 0;
510 static const char * const m[]={
511 NULL,
512 "timeout",
513 "connection refused",
514 "HTTP error",
515 "FTP error"
516 };
517
518 sleeptime = per->retry_sleep;
519 if(RETRY_HTTP == retry) {
520 curl_easy_getinfo(curl, CURLINFO_RETRY_AFTER, &retry_after);
521 if(retry_after) {
522 /* store in a 'long', make sure it doesn't overflow */
523 if(retry_after > LONG_MAX/1000)
524 sleeptime = LONG_MAX;
525 else
526 sleeptime = (long)retry_after * 1000; /* milliseconds */
527 }
528 }
529 warnf(config->global, "Transient problem: %s "
530 "Will retry in %ld seconds. "
531 "%ld retries left.\n",
532 m[retry], sleeptime/1000L, per->retry_numretries);
533
534 per->retry_numretries--;
535 tool_go_sleep(sleeptime);
536 if(!config->retry_delay) {
537 per->retry_sleep *= 2;
538 if(per->retry_sleep > RETRY_SLEEP_MAX)
539 per->retry_sleep = RETRY_SLEEP_MAX;
540 }
541 if(outs->bytes && outs->filename && outs->stream) {
542 int rc;
543 /* We have written data to a output file, we truncate file
544 */
545 if(!global->mute)
546 fprintf(global->errors, "Throwing away %"
547 CURL_FORMAT_CURL_OFF_T " bytes\n",
548 outs->bytes);
549 fflush(outs->stream);
550 /* truncate file at the position where we started appending */
551 #ifdef HAVE_FTRUNCATE
552 if(ftruncate(fileno(outs->stream), outs->init)) {
553 /* when truncate fails, we can't just append as then we'll
554 create something strange, bail out */
555 if(!global->mute)
556 fprintf(global->errors,
557 "failed to truncate, exiting\n");
558 return CURLE_WRITE_ERROR;
559 }
560 /* now seek to the end of the file, the position where we
561 just truncated the file in a large file-safe way */
562 rc = fseek(outs->stream, 0, SEEK_END);
563 #else
564 /* ftruncate is not available, so just reposition the file
565 to the location we would have truncated it. This won't
566 work properly with large files on 32-bit systems, but
567 most of those will have ftruncate. */
568 rc = fseek(outs->stream, (long)outs->init, SEEK_SET);
569 #endif
570 if(rc) {
571 if(!global->mute)
572 fprintf(global->errors,
573 "failed seeking to end of file, exiting\n");
574 return CURLE_WRITE_ERROR;
575 }
576 outs->bytes = 0; /* clear for next round */
577 }
578 *retryp = TRUE; /* curl_easy_perform loop */
579 return CURLE_OK;
580 }
581 } /* if retry_numretries */
582 else if(per->metalink) {
583 /* Metalink: Decide to try the next resource or not. Try the next resource
584 if download was not successful. */
585 long response;
586 if(CURLE_OK == result) {
587 /* TODO We want to try next resource when download was
588 not successful. How to know that? */
589 char *effective_url = NULL;
590 curl_easy_getinfo(curl, CURLINFO_EFFECTIVE_URL, &effective_url);
591 if(effective_url &&
592 curl_strnequal(effective_url, "http", 4)) {
593 /* This was HTTP(S) */
594 curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &response);
595 if(response != 200 && response != 206) {
596 per->metalink_next_res = 1;
597 fprintf(global->errors,
598 "Metalink: fetching (%s) from (%s) FAILED "
599 "(HTTP status code %ld)\n",
600 per->mlfile->filename, per->this_url, response);
601 }
602 }
603 }
604 else {
605 per->metalink_next_res = 1;
606 fprintf(global->errors,
607 "Metalink: fetching (%s) from (%s) FAILED (%s)\n",
608 per->mlfile->filename, per->this_url,
609 curl_easy_strerror(result));
610 }
611 }
612
613 if((global->progressmode == CURL_PROGRESS_BAR) &&
614 per->progressbar.calls)
615 /* if the custom progress bar has been displayed, we output a
616 newline here */
617 fputs("\n", per->progressbar.out);
618
619 if(config->writeout)
620 ourWriteOut(per->curl, &per->outs, config->writeout);
621
622 /* Close the outs file */
623 if(outs->fopened && outs->stream) {
624 int rc = fclose(outs->stream);
625 if(!result && rc) {
626 /* something went wrong in the writing process */
627 result = CURLE_WRITE_ERROR;
628 fprintf(global->errors, "(%d) Failed writing body\n", result);
629 }
630 }
631
632 /* File time can only be set _after_ the file has been closed */
633 if(!result && config->remote_time && outs->s_isreg && outs->filename) {
634 /* Ask libcurl if we got a remote file time */
635 curl_off_t filetime = -1;
636 curl_easy_getinfo(curl, CURLINFO_FILETIME_T, &filetime);
637 setfiletime(filetime, outs->filename, config->global->errors);
638 }
639
640 /* Close function-local opened file descriptors */
641 if(per->heads.fopened && per->heads.stream)
642 fclose(per->heads.stream);
643
644 if(per->heads.alloc_filename)
645 Curl_safefree(per->heads.filename);
646
647 if(per->etag_save.fopened && per->etag_save.stream)
648 fclose(per->etag_save.stream);
649
650 if(per->etag_save.alloc_filename)
651 Curl_safefree(per->etag_save.filename);
652
653 curl_easy_cleanup(per->curl);
654 if(outs->alloc_filename)
655 free(outs->filename);
656 free(per->this_url);
657 free(per->separator_err);
658 free(per->separator);
659 free(per->outfile);
660 free(per->uploadfile);
661
662 return CURLE_OK;
663 }
664
665 static void single_transfer_cleanup(struct OperationConfig *config)
666 {
667 if(config) {
668 struct State *state = &config->state;
669 if(state->urls) {
670 /* Free list of remaining URLs */
671 glob_cleanup(state->urls);
672 state->urls = NULL;
673 }
674 Curl_safefree(state->outfiles);
675 Curl_safefree(state->httpgetfields);
676 Curl_safefree(state->uploadfile);
677 if(state->inglob) {
678 /* Free list of globbed upload files */
679 glob_cleanup(state->inglob);
680 state->inglob = NULL;
681 }
682 }
683 }
684
685 /* create the next (singular) transfer */
686
687 static CURLcode single_transfer(struct GlobalConfig *global,
688 struct OperationConfig *config,
689 CURLSH *share,
690 bool capath_from_env,
691 bool *added)
692 {
693 CURLcode result = CURLE_OK;
694 struct getout *urlnode;
695 metalinkfile *mlfile_last = NULL;
696 bool orig_noprogress = global->noprogress;
697 bool orig_isatty = global->isatty;
698 struct State *state = &config->state;
699 char *httpgetfields = state->httpgetfields;
700 *added = FALSE; /* not yet */
701
702 if(config->postfields) {
703 if(config->use_httpget) {
704 if(!httpgetfields) {
705 /* Use the postfields data for a http get */
706 httpgetfields = state->httpgetfields = strdup(config->postfields);
707 Curl_safefree(config->postfields);
708 if(!httpgetfields) {
709 errorf(global, "out of memory\n");
710 result = CURLE_OUT_OF_MEMORY;
711 }
712 else if(SetHTTPrequest(config,
713 (config->no_body?HTTPREQ_HEAD:HTTPREQ_GET),
714 &config->httpreq)) {
715 result = CURLE_FAILED_INIT;
716 }
717 }
718 }
719 else {
720 if(SetHTTPrequest(config, HTTPREQ_SIMPLEPOST, &config->httpreq))
721 result = CURLE_FAILED_INIT;
722 }
723 if(result) {
724 single_transfer_cleanup(config);
725 return result;
726 }
727 }
728 if(!state->urlnode) {
729 /* first time caller, setup things */
730 state->urlnode = config->url_list;
731 state->infilenum = 1;
732 }
733
734 while(config->state.urlnode) {
735 char *infiles; /* might be a glob pattern */
736 URLGlob *inglob = state->inglob;
737 bool metalink = FALSE; /* metalink download? */
738 metalinkfile *mlfile;
739 metalink_resource *mlres;
740
741 urlnode = config->state.urlnode;
742 if(urlnode->flags & GETOUT_METALINK) {
743 metalink = 1;
744 if(mlfile_last == NULL) {
745 mlfile_last = config->metalinkfile_list;
746 }
747 mlfile = mlfile_last;
748 mlfile_last = mlfile_last->next;
749 mlres = mlfile->resource;
750 }
751 else {
752 mlfile = NULL;
753 mlres = NULL;
754 }
755
756 /* urlnode->url is the full URL (it might be NULL) */
757
758 if(!urlnode->url) {
759 /* This node has no URL. Free node data without destroying the
760 node itself nor modifying next pointer and continue to next */
761 Curl_safefree(urlnode->outfile);
762 Curl_safefree(urlnode->infile);
763 urlnode->flags = 0;
764 config->state.urlnode = urlnode->next;
765 state->up = 0;
766 continue; /* next URL please */
767 }
768
769 /* save outfile pattern before expansion */
770 if(urlnode->outfile && !state->outfiles) {
771 state->outfiles = strdup(urlnode->outfile);
772 if(!state->outfiles) {
773 errorf(global, "out of memory\n");
774 result = CURLE_OUT_OF_MEMORY;
775 break;
776 }
777 }
778
779 infiles = urlnode->infile;
780
781 if(!config->globoff && infiles && !inglob) {
782 /* Unless explicitly shut off */
783 result = glob_url(&inglob, infiles, &state->infilenum,
784 global->showerror?global->errors:NULL);
785 if(result)
786 break;
787 config->state.inglob = inglob;
788 }
789
790 {
791 int separator;
792 unsigned long urlnum;
793
794 if(!state->up && !infiles)
795 Curl_nop_stmt;
796 else {
797 if(!state->uploadfile) {
798 if(inglob) {
799 result = glob_next_url(&state->uploadfile, inglob);
800 if(result == CURLE_OUT_OF_MEMORY)
801 errorf(global, "out of memory\n");
802 }
803 else if(!state->up) {
804 state->uploadfile = strdup(infiles);
805 if(!state->uploadfile) {
806 errorf(global, "out of memory\n");
807 result = CURLE_OUT_OF_MEMORY;
808 }
809 }
810 }
811 if(result)
812 break;
813 }
814
815 if(!state->urlnum) {
816 if(metalink) {
817 /* For Metalink download, we don't use glob. Instead we use
818 the number of resources as urlnum. */
819 urlnum = count_next_metalink_resource(mlfile);
820 }
821 else if(!config->globoff) {
822 /* Unless explicitly shut off, we expand '{...}' and '[...]'
823 expressions and return total number of URLs in pattern set */
824 result = glob_url(&state->urls, urlnode->url, &state->urlnum,
825 global->showerror?global->errors:NULL);
826 if(result)
827 break;
828 urlnum = state->urlnum;
829 }
830 else
831 urlnum = 1; /* without globbing, this is a single URL */
832 }
833 else
834 urlnum = state->urlnum;
835
836 /* if multiple files extracted to stdout, insert separators! */
837 separator = ((!state->outfiles ||
838 !strcmp(state->outfiles, "-")) && urlnum > 1);
839
840 if(state->up < state->infilenum) {
841 struct per_transfer *per;
842 struct OutStruct *outs;
843 struct InStruct *input;
844 struct OutStruct *heads;
845 struct OutStruct *etag_save;
846 struct HdrCbData *hdrcbdata = NULL;
847 CURL *curl = curl_easy_init();
848 result = add_per_transfer(&per);
849 if(result || !curl) {
850 curl_easy_cleanup(curl);
851 result = CURLE_OUT_OF_MEMORY;
852 break;
853 }
854 if(state->uploadfile) {
855 per->uploadfile = strdup(state->uploadfile);
856 if(!per->uploadfile) {
857 curl_easy_cleanup(curl);
858 result = CURLE_OUT_OF_MEMORY;
859 break;
860 }
861 }
862 *added = TRUE;
863 per->config = config;
864 per->curl = curl;
865
866 /* default headers output stream is stdout */
867 heads = &per->heads;
868 heads->stream = stdout;
869
870 /* Single header file for all URLs */
871 if(config->headerfile) {
872 /* open file for output: */
873 if(strcmp(config->headerfile, "-")) {
874 FILE *newfile;
875 newfile = fopen(config->headerfile, per->prev == NULL?"wb":"ab");
876 if(!newfile) {
877 warnf(config->global, "Failed to open %s\n", config->headerfile);
878 result = CURLE_WRITE_ERROR;
879 break;
880 }
881 else {
882 heads->filename = config->headerfile;
883 heads->s_isreg = TRUE;
884 heads->fopened = TRUE;
885 heads->stream = newfile;
886 }
887 }
888 else {
889 /* always use binary mode for protocol header output */
890 set_binmode(heads->stream);
891 }
892 }
893
894 hdrcbdata = &per->hdrcbdata;
895
896 outs = &per->outs;
897 input = &per->input;
898
899 per->outfile = NULL;
900 per->infdopen = FALSE;
901 per->infd = STDIN_FILENO;
902
903 /* default output stream is stdout */
904 outs->stream = stdout;
905
906 /* --etag-save */
907 etag_save = &per->etag_save;
908 etag_save->stream = stdout;
909
910 if(config->etag_save_file) {
911 /* open file for output: */
912 if(strcmp(config->etag_save_file, "-")) {
913 FILE *newfile = fopen(config->etag_save_file, "wb");
914 if(!newfile) {
915 warnf(
916 config->global,
917 "Failed to open %s\n", config->etag_save_file);
918
919 result = CURLE_WRITE_ERROR;
920 break;
921 }
922 else {
923 etag_save->filename = config->etag_save_file;
924 etag_save->s_isreg = TRUE;
925 etag_save->fopened = TRUE;
926 etag_save->stream = newfile;
927 }
928 }
929 else {
930 /* always use binary mode for protocol header output */
931 set_binmode(etag_save->stream);
932 }
933 }
934
935 /* --etag-compare */
936 if(config->etag_compare_file) {
937 char *etag_from_file = NULL;
938 char *header = NULL;
939
940 /* open file for reading: */
941 FILE *file = fopen(config->etag_compare_file, FOPEN_READTEXT);
942 if(!file) {
943 errorf(config->global,
944 "Failed to open %s\n", config->etag_compare_file);
945 result = CURLE_READ_ERROR;
946 break;
947 }
948
949 if((PARAM_OK == file2string(&etag_from_file, file)) &&
950 etag_from_file) {
951 header = aprintf("If-None-Match: \"%s\"", etag_from_file);
952 Curl_safefree(etag_from_file);
953 }
954 else
955 header = aprintf("If-None-Match: \"\"");
956
957 if(!header) {
958 if(file)
959 fclose(file);
960 errorf(config->global,
961 "Failed to allocate memory for custom etag header\n");
962 result = CURLE_OUT_OF_MEMORY;
963 break;
964 }
965
966 /* add Etag from file to list of custom headers */
967 add2list(&config->headers, header);
968
969 Curl_safefree(header);
970
971 if(file) {
972 fclose(file);
973 }
974 }
975
976 if(metalink) {
977 /* For Metalink download, use name in Metalink file as
978 filename. */
979 per->outfile = strdup(mlfile->filename);
980 if(!per->outfile) {
981 result = CURLE_OUT_OF_MEMORY;
982 break;
983 }
984 per->this_url = strdup(mlres->url);
985 if(!per->this_url) {
986 result = CURLE_OUT_OF_MEMORY;
987 break;
988 }
989 per->mlfile = mlfile;
990 }
991 else {
992 if(state->urls) {
993 result = glob_next_url(&per->this_url, state->urls);
994 if(result)
995 break;
996 }
997 else if(!state->li) {
998 per->this_url = strdup(urlnode->url);
999 if(!per->this_url) {
1000 result = CURLE_OUT_OF_MEMORY;
1001 break;
1002 }
1003 }
1004 else
1005 per->this_url = NULL;
1006 if(!per->this_url)
1007 break;
1008
1009 if(state->outfiles) {
1010 per->outfile = strdup(state->outfiles);
1011 if(!per->outfile) {
1012 result = CURLE_OUT_OF_MEMORY;
1013 break;
1014 }
1015 }
1016 }
1017
1018 if(((urlnode->flags&GETOUT_USEREMOTE) ||
1019 (per->outfile && strcmp("-", per->outfile))) &&
1020 (metalink || !config->use_metalink)) {
1021
1022 /*
1023 * We have specified a file name to store the result in, or we have
1024 * decided we want to use the remote file name.
1025 */
1026
1027 if(!per->outfile) {
1028 /* extract the file name from the URL */
1029 result = get_url_file_name(&per->outfile, per->this_url);
1030 if(result)
1031 break;
1032 if(!*per->outfile && !config->content_disposition) {
1033 errorf(global, "Remote file name has no length!\n");
1034 result = CURLE_WRITE_ERROR;
1035 break;
1036 }
1037 }
1038 else if(state->urls) {
1039 /* fill '#1' ... '#9' terms from URL pattern */
1040 char *storefile = per->outfile;
1041 result = glob_match_url(&per->outfile, storefile, state->urls);
1042 Curl_safefree(storefile);
1043 if(result) {
1044 /* bad globbing */
1045 warnf(config->global, "bad output glob!\n");
1046 break;
1047 }
1048 }
1049
1050 /* Create the directory hierarchy, if not pre-existent to a multiple
1051 file output call */
1052
1053 if(config->create_dirs || metalink) {
1054 result = create_dir_hierarchy(per->outfile, global->errors);
1055 /* create_dir_hierarchy shows error upon CURLE_WRITE_ERROR */
1056 if(result)
1057 break;
1058 }
1059
1060 if((urlnode->flags & GETOUT_USEREMOTE)
1061 && config->content_disposition) {
1062 /* Our header callback MIGHT set the filename */
1063 DEBUGASSERT(!outs->filename);
1064 }
1065
1066 if(config->resume_from_current) {
1067 /* We're told to continue from where we are now. Get the size
1068 of the file as it is now and open it for append instead */
1069 struct_stat fileinfo;
1070 /* VMS -- Danger, the filesize is only valid for stream files */
1071 if(0 == stat(per->outfile, &fileinfo))
1072 /* set offset to current file size: */
1073 config->resume_from = fileinfo.st_size;
1074 else
1075 /* let offset be 0 */
1076 config->resume_from = 0;
1077 }
1078
1079 if(config->resume_from) {
1080 #ifdef __VMS
1081 /* open file for output, forcing VMS output format into stream
1082 mode which is needed for stat() call above to always work. */
1083 FILE *file = fopen(outfile, "ab",
1084 "ctx=stm", "rfm=stmlf", "rat=cr", "mrs=0");
1085 #else
1086 /* open file for output: */
1087 FILE *file = fopen(per->outfile, "ab");
1088 #endif
1089 if(!file) {
1090 errorf(global, "Can't open '%s'!\n", per->outfile);
1091 result = CURLE_WRITE_ERROR;
1092 break;
1093 }
1094 outs->fopened = TRUE;
1095 outs->stream = file;
1096 outs->init = config->resume_from;
1097 }
1098 else {
1099 outs->stream = NULL; /* open when needed */
1100 }
1101 outs->filename = per->outfile;
1102 outs->s_isreg = TRUE;
1103 }
1104
1105 if(per->uploadfile && !stdin_upload(per->uploadfile)) {
1106 /*
1107 * We have specified a file to upload and it isn't "-".
1108 */
1109 char *nurl = add_file_name_to_url(per->this_url, per->uploadfile);
1110 if(!nurl) {
1111 result = CURLE_OUT_OF_MEMORY;
1112 break;
1113 }
1114 per->this_url = nurl;
1115 }
1116 else if(per->uploadfile && stdin_upload(per->uploadfile)) {
1117 /* count to see if there are more than one auth bit set
1118 in the authtype field */
1119 int authbits = 0;
1120 int bitcheck = 0;
1121 while(bitcheck < 32) {
1122 if(config->authtype & (1UL << bitcheck++)) {
1123 authbits++;
1124 if(authbits > 1) {
1125 /* more than one, we're done! */
1126 break;
1127 }
1128 }
1129 }
1130
1131 /*
1132 * If the user has also selected --anyauth or --proxy-anyauth
1133 * we should warn him/her.
1134 */
1135 if(config->proxyanyauth || (authbits>1)) {
1136 warnf(config->global,
1137 "Using --anyauth or --proxy-anyauth with upload from stdin"
1138 " involves a big risk of it not working. Use a temporary"
1139 " file or a fixed auth type instead!\n");
1140 }
1141
1142 DEBUGASSERT(per->infdopen == FALSE);
1143 DEBUGASSERT(per->infd == STDIN_FILENO);
1144
1145 set_binmode(stdin);
1146 if(!strcmp(per->uploadfile, ".")) {
1147 if(curlx_nonblock((curl_socket_t)per->infd, TRUE) < 0)
1148 warnf(config->global,
1149 "fcntl failed on fd=%d: %s\n", per->infd, strerror(errno));
1150 }
1151 }
1152
1153 if(per->uploadfile && config->resume_from_current)
1154 config->resume_from = -1; /* -1 will then force get-it-yourself */
1155
1156 if(output_expected(per->this_url, per->uploadfile) && outs->stream &&
1157 isatty(fileno(outs->stream)))
1158 /* we send the output to a tty, therefore we switch off the progress
1159 meter */
1160 per->noprogress = global->noprogress = global->isatty = TRUE;
1161 else {
1162 /* progress meter is per download, so restore config
1163 values */
1164 per->noprogress = global->noprogress = orig_noprogress;
1165 global->isatty = orig_isatty;
1166 }
1167
1168 if(urlnum > 1 && !global->mute) {
1169 per->separator_err =
1170 aprintf("\n[%lu/%lu]: %s --> %s",
1171 state->li + 1, urlnum, per->this_url,
1172 per->outfile ? per->outfile : "<stdout>");
1173 if(separator)
1174 per->separator = aprintf("%s%s", CURLseparator, per->this_url);
1175 }
1176 if(httpgetfields) {
1177 char *urlbuffer;
1178 /* Find out whether the url contains a file name */
1179 const char *pc = strstr(per->this_url, "://");
1180 char sep = '?';
1181 if(pc)
1182 pc += 3;
1183 else
1184 pc = per->this_url;
1185
1186 pc = strrchr(pc, '/'); /* check for a slash */
1187
1188 if(pc) {
1189 /* there is a slash present in the URL */
1190
1191 if(strchr(pc, '?'))
1192 /* Ouch, there's already a question mark in the URL string, we
1193 then append the data with an ampersand separator instead! */
1194 sep = '&';
1195 }
1196 /*
1197 * Then append ? followed by the get fields to the url.
1198 */
1199 if(pc)
1200 urlbuffer = aprintf("%s%c%s", per->this_url, sep, httpgetfields);
1201 else
1202 /* Append / before the ? to create a well-formed url
1203 if the url contains a hostname only
1204 */
1205 urlbuffer = aprintf("%s/?%s", per->this_url, httpgetfields);
1206
1207 if(!urlbuffer) {
1208 result = CURLE_OUT_OF_MEMORY;
1209 break;
1210 }
1211
1212 Curl_safefree(per->this_url); /* free previous URL */
1213 per->this_url = urlbuffer; /* use our new URL instead! */
1214 }
1215
1216 if(!global->errors)
1217 global->errors = stderr;
1218
1219 if((!per->outfile || !strcmp(per->outfile, "-")) &&
1220 !config->use_ascii) {
1221 /* We get the output to stdout and we have not got the ASCII/text
1222 flag, then set stdout to be binary */
1223 set_binmode(stdout);
1224 }
1225
1226 /* explicitly passed to stdout means okaying binary gunk */
1227 config->terminal_binary_ok =
1228 (per->outfile && !strcmp(per->outfile, "-"));
1229
1230 /* Avoid having this setopt added to the --libcurl source output. */
1231 result = curl_easy_setopt(curl, CURLOPT_SHARE, share);
1232 if(result)
1233 break;
1234
1235 if(!config->tcp_nodelay)
1236 my_setopt(curl, CURLOPT_TCP_NODELAY, 0L);
1237
1238 if(config->tcp_fastopen)
1239 my_setopt(curl, CURLOPT_TCP_FASTOPEN, 1L);
1240
1241 /* where to store */
1242 my_setopt(curl, CURLOPT_WRITEDATA, per);
1243 my_setopt(curl, CURLOPT_INTERLEAVEDATA, per);
1244
1245 if(metalink || !config->use_metalink)
1246 /* what call to write */
1247 my_setopt(curl, CURLOPT_WRITEFUNCTION, tool_write_cb);
1248 #ifdef USE_METALINK
1249 else
1250 /* Set Metalink specific write callback function to parse
1251 XML data progressively. */
1252 my_setopt(curl, CURLOPT_WRITEFUNCTION, metalink_write_cb);
1253 #endif /* USE_METALINK */
1254
1255 /* for uploads */
1256 input->config = config;
1257 /* Note that if CURLOPT_READFUNCTION is fread (the default), then
1258 * lib/telnet.c will Curl_poll() on the input file descriptor
1259 * rather then calling the READFUNCTION at regular intervals.
1260 * The circumstances in which it is preferable to enable this
1261 * behaviour, by omitting to set the READFUNCTION & READDATA options,
1262 * have not been determined.
1263 */
1264 my_setopt(curl, CURLOPT_READDATA, input);
1265 /* what call to read */
1266 my_setopt(curl, CURLOPT_READFUNCTION, tool_read_cb);
1267
1268 /* in 7.18.0, the CURLOPT_SEEKFUNCTION/DATA pair is taking over what
1269 CURLOPT_IOCTLFUNCTION/DATA pair previously provided for seeking */
1270 my_setopt(curl, CURLOPT_SEEKDATA, input);
1271 my_setopt(curl, CURLOPT_SEEKFUNCTION, tool_seek_cb);
1272
1273 if(config->recvpersecond &&
1274 (config->recvpersecond < BUFFER_SIZE))
1275 /* use a smaller sized buffer for better sleeps */
1276 my_setopt(curl, CURLOPT_BUFFERSIZE, (long)config->recvpersecond);
1277 else
1278 my_setopt(curl, CURLOPT_BUFFERSIZE, (long)BUFFER_SIZE);
1279
1280 my_setopt_str(curl, CURLOPT_URL, per->this_url);
1281 my_setopt(curl, CURLOPT_NOPROGRESS, global->noprogress?1L:0L);
1282 if(config->no_body)
1283 my_setopt(curl, CURLOPT_NOBODY, 1L);
1284
1285 if(config->oauth_bearer)
1286 my_setopt_str(curl, CURLOPT_XOAUTH2_BEARER, config->oauth_bearer);
1287
1288 {
1289 my_setopt_str(curl, CURLOPT_PROXY, config->proxy);
1290 /* new in libcurl 7.5 */
1291 if(config->proxy)
1292 my_setopt_enum(curl, CURLOPT_PROXYTYPE, config->proxyver);
1293
1294 my_setopt_str(curl, CURLOPT_PROXYUSERPWD, config->proxyuserpwd);
1295
1296 /* new in libcurl 7.3 */
1297 my_setopt(curl, CURLOPT_HTTPPROXYTUNNEL, config->proxytunnel?1L:0L);
1298
1299 /* new in libcurl 7.52.0 */
1300 if(config->preproxy)
1301 my_setopt_str(curl, CURLOPT_PRE_PROXY, config->preproxy);
1302
1303 /* new in libcurl 7.10.6 */
1304 if(config->proxyanyauth)
1305 my_setopt_bitmask(curl, CURLOPT_PROXYAUTH,
1306 (long)CURLAUTH_ANY);
1307 else if(config->proxynegotiate)
1308 my_setopt_bitmask(curl, CURLOPT_PROXYAUTH,
1309 (long)CURLAUTH_GSSNEGOTIATE);
1310 else if(config->proxyntlm)
1311 my_setopt_bitmask(curl, CURLOPT_PROXYAUTH,
1312 (long)CURLAUTH_NTLM);
1313 else if(config->proxydigest)
1314 my_setopt_bitmask(curl, CURLOPT_PROXYAUTH,
1315 (long)CURLAUTH_DIGEST);
1316 else if(config->proxybasic)
1317 my_setopt_bitmask(curl, CURLOPT_PROXYAUTH,
1318 (long)CURLAUTH_BASIC);
1319
1320 /* new in libcurl 7.19.4 */
1321 my_setopt_str(curl, CURLOPT_NOPROXY, config->noproxy);
1322
1323 my_setopt(curl, CURLOPT_SUPPRESS_CONNECT_HEADERS,
1324 config->suppress_connect_headers?1L:0L);
1325 }
1326
1327 my_setopt(curl, CURLOPT_FAILONERROR, config->failonerror?1L:0L);
1328 my_setopt(curl, CURLOPT_REQUEST_TARGET, config->request_target);
1329 my_setopt(curl, CURLOPT_UPLOAD, per->uploadfile?1L:0L);
1330 my_setopt(curl, CURLOPT_DIRLISTONLY, config->dirlistonly?1L:0L);
1331 my_setopt(curl, CURLOPT_APPEND, config->ftp_append?1L:0L);
1332
1333 if(config->netrc_opt)
1334 my_setopt_enum(curl, CURLOPT_NETRC, (long)CURL_NETRC_OPTIONAL);
1335 else if(config->netrc || config->netrc_file)
1336 my_setopt_enum(curl, CURLOPT_NETRC, (long)CURL_NETRC_REQUIRED);
1337 else
1338 my_setopt_enum(curl, CURLOPT_NETRC, (long)CURL_NETRC_IGNORED);
1339
1340 if(config->netrc_file)
1341 my_setopt_str(curl, CURLOPT_NETRC_FILE, config->netrc_file);
1342
1343 my_setopt(curl, CURLOPT_TRANSFERTEXT, config->use_ascii?1L:0L);
1344 if(config->login_options)
1345 my_setopt_str(curl, CURLOPT_LOGIN_OPTIONS, config->login_options);
1346 my_setopt_str(curl, CURLOPT_USERPWD, config->userpwd);
1347 my_setopt_str(curl, CURLOPT_RANGE, config->range);
1348 my_setopt(curl, CURLOPT_ERRORBUFFER, per->errorbuffer);
1349 my_setopt(curl, CURLOPT_TIMEOUT_MS, (long)(config->timeout * 1000));
1350
1351 switch(config->httpreq) {
1352 case HTTPREQ_SIMPLEPOST:
1353 my_setopt_str(curl, CURLOPT_POSTFIELDS,
1354 config->postfields);
1355 my_setopt(curl, CURLOPT_POSTFIELDSIZE_LARGE,
1356 config->postfieldsize);
1357 break;
1358 case HTTPREQ_MIMEPOST:
1359 /* free previous remainders */
1360 curl_mime_free(config->mimepost);
1361 config->mimepost = NULL;
1362 result = tool2curlmime(curl, config->mimeroot, &config->mimepost);
1363 if(result)
1364 break;
1365 my_setopt_mimepost(curl, CURLOPT_MIMEPOST, config->mimepost);
1366 break;
1367 default:
1368 break;
1369 }
1370 if(result)
1371 break;
1372
1373 /* new in libcurl 7.10.6 (default is Basic) */
1374 if(config->authtype)
1375 my_setopt_bitmask(curl, CURLOPT_HTTPAUTH, (long)config->authtype);
1376
1377 my_setopt_slist(curl, CURLOPT_HTTPHEADER, config->headers);
1378
1379 if(built_in_protos & (CURLPROTO_HTTP | CURLPROTO_RTSP)) {
1380 my_setopt_str(curl, CURLOPT_REFERER, config->referer);
1381 my_setopt_str(curl, CURLOPT_USERAGENT, config->useragent);
1382 }
1383
1384 if(built_in_protos & CURLPROTO_HTTP) {
1385
1386 long postRedir = 0;
1387
1388 my_setopt(curl, CURLOPT_FOLLOWLOCATION,
1389 config->followlocation?1L:0L);
1390 my_setopt(curl, CURLOPT_UNRESTRICTED_AUTH,
1391 config->unrestricted_auth?1L:0L);
1392
1393 my_setopt(curl, CURLOPT_AUTOREFERER, config->autoreferer?1L:0L);
1394
1395 /* new in libcurl 7.36.0 */
1396 if(config->proxyheaders) {
1397 my_setopt_slist(curl, CURLOPT_PROXYHEADER, config->proxyheaders);
1398 my_setopt(curl, CURLOPT_HEADEROPT, CURLHEADER_SEPARATE);
1399 }
1400
1401 /* new in libcurl 7.5 */
1402 my_setopt(curl, CURLOPT_MAXREDIRS, config->maxredirs);
1403
1404 if(config->httpversion)
1405 my_setopt_enum(curl, CURLOPT_HTTP_VERSION, config->httpversion);
1406 else if(curlinfo->features & CURL_VERSION_HTTP2) {
1407 my_setopt_enum(curl, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2TLS);
1408 }
1409
1410 /* curl 7.19.1 (the 301 version existed in 7.18.2),
1411 303 was added in 7.26.0 */
1412 if(config->post301)
1413 postRedir |= CURL_REDIR_POST_301;
1414 if(config->post302)
1415 postRedir |= CURL_REDIR_POST_302;
1416 if(config->post303)
1417 postRedir |= CURL_REDIR_POST_303;
1418 my_setopt(curl, CURLOPT_POSTREDIR, postRedir);
1419
1420 /* new in libcurl 7.21.6 */
1421 if(config->encoding)
1422 my_setopt_str(curl, CURLOPT_ACCEPT_ENCODING, "");
1423
1424 /* new in libcurl 7.21.6 */
1425 if(config->tr_encoding)
1426 my_setopt(curl, CURLOPT_TRANSFER_ENCODING, 1L);
1427 /* new in libcurl 7.64.0 */
1428 my_setopt(curl, CURLOPT_HTTP09_ALLOWED,
1429 config->http09_allowed ? 1L : 0L);
1430
1431 } /* (built_in_protos & CURLPROTO_HTTP) */
1432
1433 my_setopt_str(curl, CURLOPT_FTPPORT, config->ftpport);
1434 my_setopt(curl, CURLOPT_LOW_SPEED_LIMIT,
1435 config->low_speed_limit);
1436 my_setopt(curl, CURLOPT_LOW_SPEED_TIME, config->low_speed_time);
1437 my_setopt(curl, CURLOPT_MAX_SEND_SPEED_LARGE,
1438 config->sendpersecond);
1439 my_setopt(curl, CURLOPT_MAX_RECV_SPEED_LARGE,
1440 config->recvpersecond);
1441
1442 if(config->use_resume)
1443 my_setopt(curl, CURLOPT_RESUME_FROM_LARGE, config->resume_from);
1444 else
1445 my_setopt(curl, CURLOPT_RESUME_FROM_LARGE, CURL_OFF_T_C(0));
1446
1447 my_setopt_str(curl, CURLOPT_KEYPASSWD, config->key_passwd);
1448 my_setopt_str(curl, CURLOPT_PROXY_KEYPASSWD, config->proxy_key_passwd);
1449
1450 if(built_in_protos & (CURLPROTO_SCP|CURLPROTO_SFTP)) {
1451
1452 /* SSH and SSL private key uses same command-line option */
1453 /* new in libcurl 7.16.1 */
1454 my_setopt_str(curl, CURLOPT_SSH_PRIVATE_KEYFILE, config->key);
1455 /* new in libcurl 7.16.1 */
1456 my_setopt_str(curl, CURLOPT_SSH_PUBLIC_KEYFILE, config->pubkey);
1457
1458 /* new in libcurl 7.17.1: SSH host key md5 checking allows us
1459 to fail if we are not talking to who we think we should */
1460 my_setopt_str(curl, CURLOPT_SSH_HOST_PUBLIC_KEY_MD5,
1461 config->hostpubmd5);
1462
1463 /* new in libcurl 7.56.0 */
1464 if(config->ssh_compression)
1465 my_setopt(curl, CURLOPT_SSH_COMPRESSION, 1L);
1466 }
1467
1468 if(config->cacert)
1469 my_setopt_str(curl, CURLOPT_CAINFO, config->cacert);
1470 if(config->proxy_cacert)
1471 my_setopt_str(curl, CURLOPT_PROXY_CAINFO, config->proxy_cacert);
1472
1473 if(config->capath) {
1474 result = res_setopt_str(curl, CURLOPT_CAPATH, config->capath);
1475 if(result == CURLE_NOT_BUILT_IN) {
1476 warnf(config->global, "ignoring %s, not supported by libcurl\n",
1477 capath_from_env?
1478 "SSL_CERT_DIR environment variable":"--capath");
1479 }
1480 else if(result)
1481 break;
1482 }
1483 /* For the time being if --proxy-capath is not set then we use the
1484 --capath value for it, if any. See #1257 */
1485 if((config->proxy_capath || config->capath) &&
1486 !tool_setopt_skip(CURLOPT_PROXY_CAPATH)) {
1487 result = res_setopt_str(curl, CURLOPT_PROXY_CAPATH,
1488 (config->proxy_capath ?
1489 config->proxy_capath :
1490 config->capath));
1491 if(result == CURLE_NOT_BUILT_IN) {
1492 if(config->proxy_capath) {
1493 warnf(config->global,
1494 "ignoring --proxy-capath, not supported by libcurl\n");
1495 }
1496 }
1497 else if(result)
1498 break;
1499 }
1500
1501 if(config->crlfile)
1502 my_setopt_str(curl, CURLOPT_CRLFILE, config->crlfile);
1503 if(config->proxy_crlfile)
1504 my_setopt_str(curl, CURLOPT_PROXY_CRLFILE, config->proxy_crlfile);
1505 else if(config->crlfile) /* CURLOPT_PROXY_CRLFILE default is crlfile */
1506 my_setopt_str(curl, CURLOPT_PROXY_CRLFILE, config->crlfile);
1507
1508 if(config->pinnedpubkey)
1509 my_setopt_str(curl, CURLOPT_PINNEDPUBLICKEY, config->pinnedpubkey);
1510
1511 if(curlinfo->features & CURL_VERSION_SSL) {
1512 /* Check if config->cert is a PKCS#11 URI and set the
1513 * config->cert_type if necessary */
1514 if(config->cert) {
1515 if(!config->cert_type) {
1516 if(is_pkcs11_uri(config->cert)) {
1517 config->cert_type = strdup("ENG");
1518 }
1519 }
1520 }
1521
1522 /* Check if config->key is a PKCS#11 URI and set the
1523 * config->key_type if necessary */
1524 if(config->key) {
1525 if(!config->key_type) {
1526 if(is_pkcs11_uri(config->key)) {
1527 config->key_type = strdup("ENG");
1528 }
1529 }
1530 }
1531
1532 /* Check if config->proxy_cert is a PKCS#11 URI and set the
1533 * config->proxy_type if necessary */
1534 if(config->proxy_cert) {
1535 if(!config->proxy_cert_type) {
1536 if(is_pkcs11_uri(config->proxy_cert)) {
1537 config->proxy_cert_type = strdup("ENG");
1538 }
1539 }
1540 }
1541
1542 /* Check if config->proxy_key is a PKCS#11 URI and set the
1543 * config->proxy_key_type if necessary */
1544 if(config->proxy_key) {
1545 if(!config->proxy_key_type) {
1546 if(is_pkcs11_uri(config->proxy_key)) {
1547 config->proxy_key_type = strdup("ENG");
1548 }
1549 }
1550 }
1551
1552 my_setopt_str(curl, CURLOPT_SSLCERT, config->cert);
1553 my_setopt_str(curl, CURLOPT_PROXY_SSLCERT, config->proxy_cert);
1554 my_setopt_str(curl, CURLOPT_SSLCERTTYPE, config->cert_type);
1555 my_setopt_str(curl, CURLOPT_PROXY_SSLCERTTYPE,
1556 config->proxy_cert_type);
1557 my_setopt_str(curl, CURLOPT_SSLKEY, config->key);
1558 my_setopt_str(curl, CURLOPT_PROXY_SSLKEY, config->proxy_key);
1559 my_setopt_str(curl, CURLOPT_SSLKEYTYPE, config->key_type);
1560 my_setopt_str(curl, CURLOPT_PROXY_SSLKEYTYPE,
1561 config->proxy_key_type);
1562
1563 if(config->insecure_ok) {
1564 my_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0L);
1565 my_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0L);
1566 }
1567 else {
1568 my_setopt(curl, CURLOPT_SSL_VERIFYPEER, 1L);
1569 /* libcurl default is strict verifyhost -> 2L */
1570 /* my_setopt(curl, CURLOPT_SSL_VERIFYHOST, 2L); */
1571 }
1572 if(config->proxy_insecure_ok) {
1573 my_setopt(curl, CURLOPT_PROXY_SSL_VERIFYPEER, 0L);
1574 my_setopt(curl, CURLOPT_PROXY_SSL_VERIFYHOST, 0L);
1575 }
1576 else {
1577 my_setopt(curl, CURLOPT_PROXY_SSL_VERIFYPEER, 1L);
1578 }
1579
1580 if(config->verifystatus)
1581 my_setopt(curl, CURLOPT_SSL_VERIFYSTATUS, 1L);
1582
1583 if(config->falsestart)
1584 my_setopt(curl, CURLOPT_SSL_FALSESTART, 1L);
1585
1586 my_setopt_enum(curl, CURLOPT_SSLVERSION,
1587 config->ssl_version | config->ssl_version_max);
1588 my_setopt_enum(curl, CURLOPT_PROXY_SSLVERSION,
1589 config->proxy_ssl_version);
1590 }
1591 if(config->path_as_is)
1592 my_setopt(curl, CURLOPT_PATH_AS_IS, 1L);
1593
1594 if(built_in_protos & (CURLPROTO_SCP|CURLPROTO_SFTP)) {
1595 if(!config->insecure_ok) {
1596 char *home;
1597 char *file;
1598 result = CURLE_FAILED_INIT;
1599 home = homedir();
1600 if(home) {
1601 file = aprintf("%s/.ssh/known_hosts", home);
1602 if(file) {
1603 /* new in curl 7.19.6 */
1604 result = res_setopt_str(curl, CURLOPT_SSH_KNOWNHOSTS, file);
1605 curl_free(file);
1606 if(result == CURLE_UNKNOWN_OPTION)
1607 /* libssh2 version older than 1.1.1 */
1608 result = CURLE_OK;
1609 }
1610 Curl_safefree(home);
1611 }
1612 else {
1613 errorf(global, "Failed to figure out user's home dir!");
1614 }
1615 if(result)
1616 break;
1617 }
1618 }
1619
1620 if(config->no_body || config->remote_time) {
1621 /* no body or use remote time */
1622 my_setopt(curl, CURLOPT_FILETIME, 1L);
1623 }
1624
1625 my_setopt(curl, CURLOPT_CRLF, config->crlf?1L:0L);
1626 my_setopt_slist(curl, CURLOPT_QUOTE, config->quote);
1627 my_setopt_slist(curl, CURLOPT_POSTQUOTE, config->postquote);
1628 my_setopt_slist(curl, CURLOPT_PREQUOTE, config->prequote);
1629
1630 if(config->cookie)
1631 my_setopt_str(curl, CURLOPT_COOKIE, config->cookie);
1632
1633 if(config->cookiefile)
1634 my_setopt_str(curl, CURLOPT_COOKIEFILE, config->cookiefile);
1635
1636 /* new in libcurl 7.9 */
1637 if(config->cookiejar)
1638 my_setopt_str(curl, CURLOPT_COOKIEJAR, config->cookiejar);
1639
1640 /* new in libcurl 7.9.7 */
1641 my_setopt(curl, CURLOPT_COOKIESESSION, config->cookiesession?1L:0L);
1642
1643 my_setopt_enum(curl, CURLOPT_TIMECONDITION, (long)config->timecond);
1644 my_setopt(curl, CURLOPT_TIMEVALUE_LARGE, config->condtime);
1645 my_setopt_str(curl, CURLOPT_CUSTOMREQUEST, config->customrequest);
1646 customrequest_helper(config, config->httpreq, config->customrequest);
1647 my_setopt(curl, CURLOPT_STDERR, global->errors);
1648
1649 /* three new ones in libcurl 7.3: */
1650 my_setopt_str(curl, CURLOPT_INTERFACE, config->iface);
1651 my_setopt_str(curl, CURLOPT_KRBLEVEL, config->krblevel);
1652 progressbarinit(&per->progressbar, config);
1653
1654 if((global->progressmode == CURL_PROGRESS_BAR) &&
1655 !global->noprogress && !global->mute) {
1656 /* we want the alternative style, then we have to implement it
1657 ourselves! */
1658 my_setopt(curl, CURLOPT_XFERINFOFUNCTION, tool_progress_cb);
1659 my_setopt(curl, CURLOPT_XFERINFODATA, per);
1660 }
1661 else if(per->uploadfile && !strcmp(per->uploadfile, ".")) {
1662 /* when reading from stdin in non-blocking mode, we use the progress
1663 function to unpause a busy read */
1664 my_setopt(curl, CURLOPT_NOPROGRESS, 0L);
1665 my_setopt(curl, CURLOPT_XFERINFOFUNCTION, tool_readbusy_cb);
1666 my_setopt(curl, CURLOPT_XFERINFODATA, per);
1667 }
1668
1669 /* new in libcurl 7.24.0: */
1670 if(config->dns_servers)
1671 my_setopt_str(curl, CURLOPT_DNS_SERVERS, config->dns_servers);
1672
1673 /* new in libcurl 7.33.0: */
1674 if(config->dns_interface)
1675 my_setopt_str(curl, CURLOPT_DNS_INTERFACE, config->dns_interface);
1676 if(config->dns_ipv4_addr)
1677 my_setopt_str(curl, CURLOPT_DNS_LOCAL_IP4, config->dns_ipv4_addr);
1678 if(config->dns_ipv6_addr)
1679 my_setopt_str(curl, CURLOPT_DNS_LOCAL_IP6, config->dns_ipv6_addr);
1680
1681 /* new in libcurl 7.6.2: */
1682 my_setopt_slist(curl, CURLOPT_TELNETOPTIONS, config->telnet_options);
1683
1684 /* new in libcurl 7.7: */
1685 my_setopt_str(curl, CURLOPT_RANDOM_FILE, config->random_file);
1686 my_setopt_str(curl, CURLOPT_EGDSOCKET, config->egd_file);
1687 my_setopt(curl, CURLOPT_CONNECTTIMEOUT_MS,
1688 (long)(config->connecttimeout * 1000));
1689
1690 if(config->doh_url)
1691 my_setopt_str(curl, CURLOPT_DOH_URL, config->doh_url);
1692
1693 if(config->cipher_list)
1694 my_setopt_str(curl, CURLOPT_SSL_CIPHER_LIST, config->cipher_list);
1695
1696 if(config->proxy_cipher_list)
1697 my_setopt_str(curl, CURLOPT_PROXY_SSL_CIPHER_LIST,
1698 config->proxy_cipher_list);
1699
1700 if(config->cipher13_list)
1701 my_setopt_str(curl, CURLOPT_TLS13_CIPHERS, config->cipher13_list);
1702
1703 if(config->proxy_cipher13_list)
1704 my_setopt_str(curl, CURLOPT_PROXY_TLS13_CIPHERS,
1705 config->proxy_cipher13_list);
1706
1707 /* new in libcurl 7.9.2: */
1708 if(config->disable_epsv)
1709 /* disable it */
1710 my_setopt(curl, CURLOPT_FTP_USE_EPSV, 0L);
1711
1712 /* new in libcurl 7.10.5 */
1713 if(config->disable_eprt)
1714 /* disable it */
1715 my_setopt(curl, CURLOPT_FTP_USE_EPRT, 0L);
1716
1717 if(global->tracetype != TRACE_NONE) {
1718 my_setopt(curl, CURLOPT_DEBUGFUNCTION, tool_debug_cb);
1719 my_setopt(curl, CURLOPT_DEBUGDATA, config);
1720 my_setopt(curl, CURLOPT_VERBOSE, 1L);
1721 }
1722
1723 /* new in curl 7.9.3 */
1724 if(config->engine) {
1725 result = res_setopt_str(curl, CURLOPT_SSLENGINE, config->engine);
1726 if(result)
1727 break;
1728 }
1729
1730 /* new in curl 7.10.7, extended in 7.19.4. Modified to use
1731 CREATE_DIR_RETRY in 7.49.0 */
1732 my_setopt(curl, CURLOPT_FTP_CREATE_MISSING_DIRS,
1733 (long)(config->ftp_create_dirs?
1734 CURLFTP_CREATE_DIR_RETRY:
1735 CURLFTP_CREATE_DIR_NONE));
1736
1737 /* new in curl 7.10.8 */
1738 if(config->max_filesize)
1739 my_setopt(curl, CURLOPT_MAXFILESIZE_LARGE,
1740 config->max_filesize);
1741
1742 if(4 == config->ip_version)
1743 my_setopt(curl, CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V4);
1744 else if(6 == config->ip_version)
1745 my_setopt(curl, CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V6);
1746 else
1747 my_setopt(curl, CURLOPT_IPRESOLVE, CURL_IPRESOLVE_WHATEVER);
1748
1749 /* new in curl 7.15.5 */
1750 if(config->ftp_ssl_reqd)
1751 my_setopt_enum(curl, CURLOPT_USE_SSL, (long)CURLUSESSL_ALL);
1752
1753 /* new in curl 7.11.0 */
1754 else if(config->ftp_ssl)
1755 my_setopt_enum(curl, CURLOPT_USE_SSL, (long)CURLUSESSL_TRY);
1756
1757 /* new in curl 7.16.0 */
1758 else if(config->ftp_ssl_control)
1759 my_setopt_enum(curl, CURLOPT_USE_SSL, (long)CURLUSESSL_CONTROL);
1760
1761 /* new in curl 7.16.1 */
1762 if(config->ftp_ssl_ccc)
1763 my_setopt_enum(curl, CURLOPT_FTP_SSL_CCC,
1764 (long)config->ftp_ssl_ccc_mode);
1765
1766 /* new in curl 7.19.4 */
1767 if(config->socks5_gssapi_nec)
1768 my_setopt_str(curl, CURLOPT_SOCKS5_GSSAPI_NEC,
1769 config->socks5_gssapi_nec);
1770
1771 /* new in curl 7.55.0 */
1772 if(config->socks5_auth)
1773 my_setopt_bitmask(curl, CURLOPT_SOCKS5_AUTH,
1774 (long)config->socks5_auth);
1775
1776 /* new in curl 7.43.0 */
1777 if(config->proxy_service_name)
1778 my_setopt_str(curl, CURLOPT_PROXY_SERVICE_NAME,
1779 config->proxy_service_name);
1780
1781 /* new in curl 7.43.0 */
1782 if(config->service_name)
1783 my_setopt_str(curl, CURLOPT_SERVICE_NAME,
1784 config->service_name);
1785
1786 /* curl 7.13.0 */
1787 my_setopt_str(curl, CURLOPT_FTP_ACCOUNT, config->ftp_account);
1788 my_setopt(curl, CURLOPT_IGNORE_CONTENT_LENGTH, config->ignorecl?1L:0L);
1789
1790 /* curl 7.14.2 */
1791 my_setopt(curl, CURLOPT_FTP_SKIP_PASV_IP, config->ftp_skip_ip?1L:0L);
1792
1793 /* curl 7.15.1 */
1794 my_setopt(curl, CURLOPT_FTP_FILEMETHOD, (long)config->ftp_filemethod);
1795
1796 /* curl 7.15.2 */
1797 if(config->localport) {
1798 my_setopt(curl, CURLOPT_LOCALPORT, config->localport);
1799 my_setopt_str(curl, CURLOPT_LOCALPORTRANGE, config->localportrange);
1800 }
1801
1802 /* curl 7.15.5 */
1803 my_setopt_str(curl, CURLOPT_FTP_ALTERNATIVE_TO_USER,
1804 config->ftp_alternative_to_user);
1805
1806 /* curl 7.16.0 */
1807 if(config->disable_sessionid)
1808 /* disable it */
1809 my_setopt(curl, CURLOPT_SSL_SESSIONID_CACHE, 0L);
1810
1811 /* curl 7.16.2 */
1812 if(config->raw) {
1813 my_setopt(curl, CURLOPT_HTTP_CONTENT_DECODING, 0L);
1814 my_setopt(curl, CURLOPT_HTTP_TRANSFER_DECODING, 0L);
1815 }
1816
1817 /* curl 7.17.1 */
1818 if(!config->nokeepalive) {
1819 my_setopt(curl, CURLOPT_TCP_KEEPALIVE, 1L);
1820 if(config->alivetime != 0) {
1821 my_setopt(curl, CURLOPT_TCP_KEEPIDLE, config->alivetime);
1822 my_setopt(curl, CURLOPT_TCP_KEEPINTVL, config->alivetime);
1823 }
1824 }
1825 else
1826 my_setopt(curl, CURLOPT_TCP_KEEPALIVE, 0L);
1827
1828 /* curl 7.20.0 */
1829 if(config->tftp_blksize)
1830 my_setopt(curl, CURLOPT_TFTP_BLKSIZE, config->tftp_blksize);
1831
1832 if(config->mail_from)
1833 my_setopt_str(curl, CURLOPT_MAIL_FROM, config->mail_from);
1834
1835 if(config->mail_rcpt)
1836 my_setopt_slist(curl, CURLOPT_MAIL_RCPT, config->mail_rcpt);
1837
1838 /* curl 7.69.x */
1839 my_setopt(curl, CURLOPT_MAIL_RCPT_ALLLOWFAILS,
1840 config->mail_rcpt_allowfails ? 1L : 0L);
1841
1842 /* curl 7.20.x */
1843 if(config->ftp_pret)
1844 my_setopt(curl, CURLOPT_FTP_USE_PRET, 1L);
1845
1846 if(config->proto_present)
1847 my_setopt_flags(curl, CURLOPT_PROTOCOLS, config->proto);
1848 if(config->proto_redir_present)
1849 my_setopt_flags(curl, CURLOPT_REDIR_PROTOCOLS, config->proto_redir);
1850
1851 if(config->content_disposition
1852 && (urlnode->flags & GETOUT_USEREMOTE))
1853 hdrcbdata->honor_cd_filename = TRUE;
1854 else
1855 hdrcbdata->honor_cd_filename = FALSE;
1856
1857 hdrcbdata->outs = outs;
1858 hdrcbdata->heads = heads;
1859 hdrcbdata->etag_save = etag_save;
1860 hdrcbdata->global = global;
1861 hdrcbdata->config = config;
1862
1863 my_setopt(curl, CURLOPT_HEADERFUNCTION, tool_header_cb);
1864 my_setopt(curl, CURLOPT_HEADERDATA, per);
1865
1866 if(config->resolve)
1867 /* new in 7.21.3 */
1868 my_setopt_slist(curl, CURLOPT_RESOLVE, config->resolve);
1869
1870 if(config->connect_to)
1871 /* new in 7.49.0 */
1872 my_setopt_slist(curl, CURLOPT_CONNECT_TO, config->connect_to);
1873
1874 /* new in 7.21.4 */
1875 if(curlinfo->features & CURL_VERSION_TLSAUTH_SRP) {
1876 if(config->tls_username)
1877 my_setopt_str(curl, CURLOPT_TLSAUTH_USERNAME,
1878 config->tls_username);
1879 if(config->tls_password)
1880 my_setopt_str(curl, CURLOPT_TLSAUTH_PASSWORD,
1881 config->tls_password);
1882 if(config->tls_authtype)
1883 my_setopt_str(curl, CURLOPT_TLSAUTH_TYPE,
1884 config->tls_authtype);
1885 if(config->proxy_tls_username)
1886 my_setopt_str(curl, CURLOPT_PROXY_TLSAUTH_USERNAME,
1887 config->proxy_tls_username);
1888 if(config->proxy_tls_password)
1889 my_setopt_str(curl, CURLOPT_PROXY_TLSAUTH_PASSWORD,
1890 config->proxy_tls_password);
1891 if(config->proxy_tls_authtype)
1892 my_setopt_str(curl, CURLOPT_PROXY_TLSAUTH_TYPE,
1893 config->proxy_tls_authtype);
1894 }
1895
1896 /* new in 7.22.0 */
1897 if(config->gssapi_delegation)
1898 my_setopt_str(curl, CURLOPT_GSSAPI_DELEGATION,
1899 config->gssapi_delegation);
1900
1901 /* new in 7.25.0 and 7.44.0 */
1902 {
1903 long mask = (config->ssl_allow_beast ? CURLSSLOPT_ALLOW_BEAST : 0) |
1904 (config->ssl_no_revoke ? CURLSSLOPT_NO_REVOKE : 0);
1905 if(mask)
1906 my_setopt_bitmask(curl, CURLOPT_SSL_OPTIONS, mask);
1907 }
1908
1909 if(config->proxy_ssl_allow_beast)
1910 my_setopt(curl, CURLOPT_PROXY_SSL_OPTIONS,
1911 (long)CURLSSLOPT_ALLOW_BEAST);
1912
1913 if(config->mail_auth)
1914 my_setopt_str(curl, CURLOPT_MAIL_AUTH, config->mail_auth);
1915
1916 /* new in 7.66.0 */
1917 if(config->sasl_authzid)
1918 my_setopt_str(curl, CURLOPT_SASL_AUTHZID, config->sasl_authzid);
1919
1920 /* new in 7.31.0 */
1921 if(config->sasl_ir)
1922 my_setopt(curl, CURLOPT_SASL_IR, 1L);
1923
1924 if(config->nonpn) {
1925 my_setopt(curl, CURLOPT_SSL_ENABLE_NPN, 0L);
1926 }
1927
1928 if(config->noalpn) {
1929 my_setopt(curl, CURLOPT_SSL_ENABLE_ALPN, 0L);
1930 }
1931
1932 /* new in 7.40.0, abstract support added in 7.53.0 */
1933 if(config->unix_socket_path) {
1934 if(config->abstract_unix_socket) {
1935 my_setopt_str(curl, CURLOPT_ABSTRACT_UNIX_SOCKET,
1936 config->unix_socket_path);
1937 }
1938 else {
1939 my_setopt_str(curl, CURLOPT_UNIX_SOCKET_PATH,
1940 config->unix_socket_path);
1941 }
1942 }
1943
1944 /* new in 7.45.0 */
1945 if(config->proto_default)
1946 my_setopt_str(curl, CURLOPT_DEFAULT_PROTOCOL, config->proto_default);
1947
1948 /* new in 7.47.0 */
1949 if(config->expect100timeout > 0)
1950 my_setopt_str(curl, CURLOPT_EXPECT_100_TIMEOUT_MS,
1951 (long)(config->expect100timeout*1000));
1952
1953 /* new in 7.48.0 */
1954 if(config->tftp_no_options)
1955 my_setopt(curl, CURLOPT_TFTP_NO_OPTIONS, 1L);
1956
1957 /* new in 7.59.0 */
1958 if(config->happy_eyeballs_timeout_ms != CURL_HET_DEFAULT)
1959 my_setopt(curl, CURLOPT_HAPPY_EYEBALLS_TIMEOUT_MS,
1960 config->happy_eyeballs_timeout_ms);
1961
1962 /* new in 7.60.0 */
1963 if(config->haproxy_protocol)
1964 my_setopt(curl, CURLOPT_HAPROXYPROTOCOL, 1L);
1965
1966 if(config->disallow_username_in_url)
1967 my_setopt(curl, CURLOPT_DISALLOW_USERNAME_IN_URL, 1L);
1968
1969 if(config->altsvc)
1970 my_setopt_str(curl, CURLOPT_ALTSVC, config->altsvc);
1971
1972 #ifdef USE_METALINK
1973 if(!metalink && config->use_metalink) {
1974 outs->metalink_parser = metalink_parser_context_new();
1975 if(outs->metalink_parser == NULL) {
1976 result = CURLE_OUT_OF_MEMORY;
1977 break;
1978 }
1979 fprintf(config->global->errors,
1980 "Metalink: parsing (%s) metalink/XML...\n", per->this_url);
1981 }
1982 else if(metalink)
1983 fprintf(config->global->errors,
1984 "Metalink: fetching (%s) from (%s)...\n",
1985 mlfile->filename, per->this_url);
1986 #endif /* USE_METALINK */
1987
1988 per->metalink = metalink;
1989 /* initialize retry vars for loop below */
1990 per->retry_sleep_default = (config->retry_delay) ?
1991 config->retry_delay*1000L : RETRY_SLEEP_DEFAULT; /* ms */
1992 per->retry_numretries = config->req_retry;
1993 per->retry_sleep = per->retry_sleep_default; /* ms */
1994 per->retrystart = tvnow();
1995
1996 state->li++;
1997 /* Here's looping around each globbed URL */
1998 if(state->li >= urlnum) {
1999 state->li = 0;
2000 state->urlnum = 0; /* forced reglob of URLs */
2001 glob_cleanup(state->urls);
2002 state->urls = NULL;
2003 state->up++;
2004 Curl_safefree(state->uploadfile); /* clear it to get the next */
2005 }
2006 }
2007 else {
2008 /* Free this URL node data without destroying the
2009 the node itself nor modifying next pointer. */
2010 Curl_safefree(urlnode->outfile);
2011 Curl_safefree(urlnode->infile);
2012 urlnode->flags = 0;
2013 glob_cleanup(state->urls);
2014 state->urls = NULL;
2015 state->urlnum = 0;
2016
2017 Curl_safefree(state->outfiles);
2018 Curl_safefree(state->uploadfile);
2019 if(state->inglob) {
2020 /* Free list of globbed upload files */
2021 glob_cleanup(state->inglob);
2022 state->inglob = NULL;
2023 }
2024 config->state.urlnode = urlnode->next;
2025 state->up = 0;
2026 continue;
2027 }
2028 }
2029 break;
2030 }
2031
2032 if(!*added || result) {
2033 *added = FALSE;
2034 single_transfer_cleanup(config);
2035 }
2036 return result;
2037 }
2038
2039 static long all_added; /* number of easy handles currently added */
2040
2041 /*
2042 * add_parallel_transfers() sets 'morep' to TRUE if there are more transfers
2043 * to add even after this call returns. sets 'addedp' to TRUE if one or more
2044 * transfers were added.
2045 */
2046 static CURLcode add_parallel_transfers(struct GlobalConfig *global,
2047 CURLM *multi,
2048 CURLSH *share,
2049 bool *morep,
2050 bool *addedp)
2051 {
2052 struct per_transfer *per;
2053 CURLcode result = CURLE_OK;
2054 CURLMcode mcode;
2055 *addedp = FALSE;
2056 *morep = FALSE;
2057 result = create_transfer(global, share, addedp);
2058 if(result || !*addedp)
2059 return result;
2060 for(per = transfers; per && (all_added < global->parallel_max);
2061 per = per->next) {
2062 bool getadded = FALSE;
2063 if(per->added)
2064 /* already added */
2065 continue;
2066
2067 result = pre_transfer(global, per);
2068 if(result)
2069 break;
2070
2071 /* parallel connect means that we don't set PIPEWAIT since pipewait
2072 will make libcurl prefer multiplexing */
2073 (void)curl_easy_setopt(per->curl, CURLOPT_PIPEWAIT,
2074 global->parallel_connect ? 0L : 1L);
2075 (void)curl_easy_setopt(per->curl, CURLOPT_PRIVATE, per);
2076 (void)curl_easy_setopt(per->curl, CURLOPT_XFERINFOFUNCTION, xferinfo_cb);
2077 (void)curl_easy_setopt(per->curl, CURLOPT_XFERINFODATA, per);
2078
2079 mcode = curl_multi_add_handle(multi, per->curl);
2080 if(mcode)
2081 return CURLE_OUT_OF_MEMORY;
2082
2083 result = create_transfer(global, share, &getadded);
2084 if(result)
2085 return result;
2086 per->added = TRUE;
2087 all_added++;
2088 *addedp = TRUE;
2089 }
2090 *morep = per ? TRUE : FALSE;
2091 return CURLE_OK;
2092 }
2093
2094 static CURLcode parallel_transfers(struct GlobalConfig *global,
2095 CURLSH *share)
2096 {
2097 CURLM *multi;
2098 CURLMcode mcode = CURLM_OK;
2099 CURLcode result = CURLE_OK;
2100 int still_running = 1;
2101 struct timeval start = tvnow();
2102 bool more_transfers;
2103 bool added_transfers;
2104
2105 multi = curl_multi_init();
2106 if(!multi)
2107 return CURLE_OUT_OF_MEMORY;
2108
2109 result = add_parallel_transfers(global, multi, share,
2110 &more_transfers, &added_transfers);
2111 if(result) {
2112 curl_multi_cleanup(multi);
2113 return result;
2114 }
2115
2116 while(!mcode && (still_running || more_transfers)) {
2117 mcode = curl_multi_poll(multi, NULL, 0, 1000, NULL);
2118 if(!mcode)
2119 mcode = curl_multi_perform(multi, &still_running);
2120
2121 progress_meter(global, &start, FALSE);
2122
2123 if(!mcode) {
2124 int rc;
2125 CURLMsg *msg;
2126 bool removed = FALSE;
2127 do {
2128 msg = curl_multi_info_read(multi, &rc);
2129 if(msg) {
2130 bool retry;
2131 struct per_transfer *ended;
2132 CURL *easy = msg->easy_handle;
2133 result = msg->data.result;
2134 curl_easy_getinfo(easy, CURLINFO_PRIVATE, (void *)&ended);
2135 curl_multi_remove_handle(multi, easy);
2136
2137 result = post_per_transfer(global, ended, result, &retry);
2138 if(retry)
2139 continue;
2140 progress_finalize(ended); /* before it goes away */
2141 all_added--; /* one fewer added */
2142 removed = TRUE;
2143 (void)del_per_transfer(ended);
2144 }
2145 } while(msg);
2146 if(removed) {
2147 /* one or more transfers completed, add more! */
2148 (void)add_parallel_transfers(global, multi, share,
2149 &more_transfers,
2150 &added_transfers);
2151 if(added_transfers)
2152 /* we added new ones, make sure the loop doesn't exit yet */
2153 still_running = 1;
2154 }
2155 }
2156 }
2157
2158 (void)progress_meter(global, &start, TRUE);
2159
2160 /* Make sure to return some kind of error if there was a multi problem */
2161 if(mcode) {
2162 result = (mcode == CURLM_OUT_OF_MEMORY) ? CURLE_OUT_OF_MEMORY :
2163 /* The other multi errors should never happen, so return
2164 something suitably generic */
2165 CURLE_BAD_FUNCTION_ARGUMENT;
2166 }
2167
2168 curl_multi_cleanup(multi);
2169
2170 return result;
2171 }
2172
2173 static CURLcode serial_transfers(struct GlobalConfig *global,
2174 CURLSH *share)
2175 {
2176 CURLcode returncode = CURLE_OK;
2177 CURLcode result = CURLE_OK;
2178 struct per_transfer *per;
2179 bool added = FALSE;
2180
2181 result = create_transfer(global, share, &added);
2182 if(result || !added)
2183 return result;
2184 for(per = transfers; per;) {
2185 bool retry;
2186 bool bailout = FALSE;
2187 result = pre_transfer(global, per);
2188 if(result)
2189 break;
2190
2191 #ifndef CURL_DISABLE_LIBCURL_OPTION
2192 if(global->libcurl) {
2193 result = easysrc_perform();
2194 if(result)
2195 break;
2196 }
2197 #endif
2198 #ifdef CURLDEBUG
2199 if(global->test_event_based)
2200 result = curl_easy_perform_ev(per->curl);
2201 else
2202 #endif
2203 result = curl_easy_perform(per->curl);
2204
2205 /* store the result of the actual transfer */
2206 returncode = result;
2207
2208 result = post_per_transfer(global, per, result, &retry);
2209 if(retry)
2210 continue;
2211
2212 /* Bail out upon critical errors or --fail-early */
2213 if(result || is_fatal_error(returncode) ||
2214 (returncode && global->fail_early))
2215 bailout = TRUE;
2216 else {
2217 /* setup the next one just before we delete this */
2218 result = create_transfer(global, share, &added);
2219 if(result)
2220 bailout = TRUE;
2221 }
2222
2223 /* Release metalink related resources here */
2224 delete_metalinkfile(per->mlfile);
2225
2226 per = del_per_transfer(per);
2227
2228 if(bailout)
2229 break;
2230 }
2231 if(returncode)
2232 /* returncode errors have priority */
2233 result = returncode;
2234
2235 if(result)
2236 single_transfer_cleanup(global->current);
2237
2238 return result;
2239 }
2240
2241 /* setup a transfer for the given config */
2242 static CURLcode transfer_per_config(struct GlobalConfig *global,
2243 struct OperationConfig *config,
2244 CURLSH *share,
2245 bool *added)
2246 {
2247 CURLcode result = CURLE_OK;
2248 bool capath_from_env;
2249 *added = FALSE;
2250
2251 /* Check we have a url */
2252 if(!config->url_list || !config->url_list->url) {
2253 helpf(global->errors, "no URL specified!\n");
2254 return CURLE_FAILED_INIT;
2255 }
2256
2257 /* On WIN32 we can't set the path to curl-ca-bundle.crt
2258 * at compile time. So we look here for the file in two ways:
2259 * 1: look at the environment variable CURL_CA_BUNDLE for a path
2260 * 2: if #1 isn't found, use the windows API function SearchPath()
2261 * to find it along the app's path (includes app's dir and CWD)
2262 *
2263 * We support the environment variable thing for non-Windows platforms
2264 * too. Just for the sake of it.
2265 */
2266 capath_from_env = false;
2267 if(!config->cacert &&
2268 !config->capath &&
2269 !config->insecure_ok) {
2270 CURL *curltls = curl_easy_init();
2271 struct curl_tlssessioninfo *tls_backend_info = NULL;
2272
2273 /* With the addition of CAINFO support for Schannel, this search could find
2274 * a certificate bundle that was previously ignored. To maintain backward
2275 * compatibility, only perform this search if not using Schannel.
2276 */
2277 result = curl_easy_getinfo(curltls, CURLINFO_TLS_SSL_PTR,
2278 &tls_backend_info);
2279 if(result)
2280 return result;
2281
2282 /* Set the CA cert locations specified in the environment. For Windows if
2283 * no environment-specified filename is found then check for CA bundle
2284 * default filename curl-ca-bundle.crt in the user's PATH.
2285 *
2286 * If Schannel is the selected SSL backend then these locations are
2287 * ignored. We allow setting CA location for schannel only when explicitly
2288 * specified by the user via CURLOPT_CAINFO / --cacert.
2289 */
2290 if(tls_backend_info->backend != CURLSSLBACKEND_SCHANNEL) {
2291 char *env;
2292 env = curlx_getenv("CURL_CA_BUNDLE");
2293 if(env) {
2294 config->cacert = strdup(env);
2295 if(!config->cacert) {
2296 curl_free(env);
2297 errorf(global, "out of memory\n");
2298 return CURLE_OUT_OF_MEMORY;
2299 }
2300 }
2301 else {
2302 env = curlx_getenv("SSL_CERT_DIR");
2303 if(env) {
2304 config->capath = strdup(env);
2305 if(!config->capath) {
2306 curl_free(env);
2307 helpf(global->errors, "out of memory\n");
2308 return CURLE_OUT_OF_MEMORY;
2309 }
2310 capath_from_env = true;
2311 }
2312 else {
2313 env = curlx_getenv("SSL_CERT_FILE");
2314 if(env) {
2315 config->cacert = strdup(env);
2316 if(!config->cacert) {
2317 curl_free(env);
2318 errorf(global, "out of memory\n");
2319 return CURLE_OUT_OF_MEMORY;
2320 }
2321 }
2322 }
2323 }
2324
2325 if(env)
2326 curl_free(env);
2327 #ifdef WIN32
2328 else {
2329 result = FindWin32CACert(config, tls_backend_info->backend,
2330 "curl-ca-bundle.crt");
2331 }
2332 #endif
2333 }
2334 curl_easy_cleanup(curltls);
2335 }
2336
2337 if(!result)
2338 result = single_transfer(global, config, share, capath_from_env, added);
2339
2340 return result;
2341 }
2342
2343 /*
2344 * 'create_transfer' gets the details and sets up a new transfer if 'added'
2345 * returns TRUE.
2346 */
2347 static CURLcode create_transfer(struct GlobalConfig *global,
2348 CURLSH *share,
2349 bool *added)
2350 {
2351 CURLcode result = CURLE_OK;
2352 *added = FALSE;
2353 while(global->current) {
2354 result = transfer_per_config(global, global->current, share, added);
2355 if(!result && !*added) {
2356 /* when one set is drained, continue to next */
2357 global->current = global->current->next;
2358 continue;
2359 }
2360 break;
2361 }
2362 return result;
2363 }
2364
2365 static CURLcode run_all_transfers(struct GlobalConfig *global,
2366 CURLSH *share,
2367 CURLcode result)
2368 {
2369 /* Save the values of noprogress and isatty to restore them later on */
2370 bool orig_noprogress = global->noprogress;
2371 bool orig_isatty = global->isatty;
2372 struct per_transfer *per;
2373
2374 /* Time to actually do the transfers */
2375 if(!result) {
2376 if(global->parallel)
2377 result = parallel_transfers(global, share);
2378 else
2379 result = serial_transfers(global, share);
2380 }
2381
2382 /* cleanup if there are any left */
2383 for(per = transfers; per;) {
2384 bool retry;
2385 CURLcode result2 = post_per_transfer(global, per, result, &retry);
2386 if(!result)
2387 /* don't overwrite the original error */
2388 result = result2;
2389
2390 /* Free list of given URLs */
2391 clean_getout(per->config);
2392
2393 /* Release metalink related resources here */
2394 clean_metalink(per->config);
2395 per = del_per_transfer(per);
2396 }
2397
2398 /* Reset the global config variables */
2399 global->noprogress = orig_noprogress;
2400 global->isatty = orig_isatty;
2401
2402
2403 return result;
2404 }
2405
2406 CURLcode operate(struct GlobalConfig *global, int argc, argv_item_t argv[])
2407 {
2408 CURLcode result = CURLE_OK;
2409
2410 /* Setup proper locale from environment */
2411 #ifdef HAVE_SETLOCALE
2412 setlocale(LC_ALL, "");
2413 #endif
2414
2415 /* Parse .curlrc if necessary */
2416 if((argc == 1) ||
2417 (!curl_strequal(argv[1], "-q") &&
2418 !curl_strequal(argv[1], "--disable"))) {
2419 parseconfig(NULL, global); /* ignore possible failure */
2420
2421 /* If we had no arguments then make sure a url was specified in .curlrc */
2422 if((argc < 2) && (!global->first->url_list)) {
2423 helpf(global->errors, NULL);
2424 result = CURLE_FAILED_INIT;
2425 }
2426 }
2427
2428 if(!result) {
2429 /* Parse the command line arguments */
2430 ParameterError res = parse_args(global, argc, argv);
2431 if(res) {
2432 result = CURLE_OK;
2433
2434 /* Check if we were asked for the help */
2435 if(res == PARAM_HELP_REQUESTED)
2436 tool_help();
2437 /* Check if we were asked for the manual */
2438 else if(res == PARAM_MANUAL_REQUESTED)
2439 hugehelp();
2440 /* Check if we were asked for the version information */
2441 else if(res == PARAM_VERSION_INFO_REQUESTED)
2442 tool_version_info();
2443 /* Check if we were asked to list the SSL engines */
2444 else if(res == PARAM_ENGINES_REQUESTED)
2445 tool_list_engines();
2446 else if(res == PARAM_LIBCURL_UNSUPPORTED_PROTOCOL)
2447 result = CURLE_UNSUPPORTED_PROTOCOL;
2448 else
2449 result = CURLE_FAILED_INIT;
2450 }
2451 else {
2452 #ifndef CURL_DISABLE_LIBCURL_OPTION
2453 if(global->libcurl) {
2454 /* Initialise the libcurl source output */
2455 result = easysrc_init();
2456 }
2457 #endif
2458
2459 /* Perform the main operations */
2460 if(!result) {
2461 size_t count = 0;
2462 struct OperationConfig *operation = global->first;
2463 CURLSH *share = curl_share_init();
2464 if(!share) {
2465 #ifndef CURL_DISABLE_LIBCURL_OPTION
2466 if(global->libcurl) {
2467 /* Cleanup the libcurl source output */
2468 easysrc_cleanup();
2469 }
2470 #endif
2471 return CURLE_OUT_OF_MEMORY;
2472 }
2473
2474 curl_share_setopt(share, CURLSHOPT_SHARE, CURL_LOCK_DATA_COOKIE);
2475 curl_share_setopt(share, CURLSHOPT_SHARE, CURL_LOCK_DATA_DNS);
2476 curl_share_setopt(share, CURLSHOPT_SHARE, CURL_LOCK_DATA_SSL_SESSION);
2477 curl_share_setopt(share, CURLSHOPT_SHARE, CURL_LOCK_DATA_CONNECT);
2478 curl_share_setopt(share, CURLSHOPT_SHARE, CURL_LOCK_DATA_PSL);
2479
2480 /* Get the required arguments for each operation */
2481 do {
2482 result = get_args(operation, count++);
2483
2484 operation = operation->next;
2485 } while(!result && operation);
2486
2487 /* Set the current operation pointer */
2488 global->current = global->first;
2489
2490 /* now run! */
2491 result = run_all_transfers(global, share, result);
2492
2493 curl_share_cleanup(share);
2494 #ifndef CURL_DISABLE_LIBCURL_OPTION
2495 if(global->libcurl) {
2496 /* Cleanup the libcurl source output */
2497 easysrc_cleanup();
2498
2499 /* Dump the libcurl code if previously enabled */
2500 dumpeasysrc(global);
2501 }
2502 #endif
2503 }
2504 else
2505 errorf(global, "out of memory\n");
2506 }
2507 }
2508
2509 return result;
2510 }