Debian Codegraph @

src/tools/davix_tool_params.cpp

raw
1 /*
2 * This File is part of Davix, The IO library for HTTP based protocols
3 * Copyright (C) CERN 2013
4 * Author: Adrien Devresse <[email protected]>
5 *
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2.1 of the License, or (at your option) any later version.
10 *
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, write to the Free Software
18 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19 *
20 */
21
22 #include <davix_internal.hpp>
23 #include "davix_tool_params.hpp"
24 #include "davix_tool_util.hpp"
25 #include "davix_config_parser.hpp"
26 #include <getopt.h>
27 #include <utils/stringutils.hpp>
28 #include <utils/davix_logger.hpp>
29
30 namespace Davix{
31
32 namespace Tool{
33
34
35 const std::string scope_params = "Davix::Tools::Params";
36
37
38 #define CAPATH_OPT 1000
39 #define DEBUG_OPT 1001
40 #define USER_LOGIN 1002
41 #define USER_PASSWORD 1003
42 #define DATA_CONTENT 1004
43 #define S3_SECRET_KEY 1005
44 #define S3_ACCESS_KEY 1006
45 #define X509_PRIVATE_KEY 1007
46 #define TRACE_OPTIONS 1008
47 #define REDIRECTION_OPT 1009
48 #define METALINK_OPT 1010
49 #define CONN_TIMEOUT 1011
50 #define TIMEOUT_OPS 1012
51 #define RETRY_OPT 1013
52 #define S3_LISTING_MODE 1014
53 #define S3_MAX_KEYS 1015
54 #define RETRY_DELAY_OPT 1016
55 #define HAS_INPUT_FILE 1017
56 #define THIRD_PT_COPY_MODE 1018
57 #define DISABLE_LISTING_CAP 1019
58 #define S3_REGION 1020
59 #define S3_ALTERNATE 1021
60 #define AZURE_KEY 1022
61 #define S3_TOKEN 1023
62 #define NO_100_CONTINUE 1024
63 #define ACCEPTED_RETRY 1025
64 #define ACCEPTED_RETRY_DELAY 1026
65 #define GCLOUD_CRED_PATH 1027
66 #define OS_TOKEN 1028
67 #define OS_PROJECT_ID 1029
68 #define SWIFT_LISTING_MODE 1030
69 #define SWIFT_ACCOUNT 1031
70
71 // LONG OPTS
72
73 #define COMMON_LONG_OPTIONS \
74 {"debug", no_argument, 0, DEBUG_OPT }, \
75 {"header", required_argument, 0, 'H' }, \
76 {"help", no_argument, 0,'?'}, \
77 {"metalink", required_argument, 0, METALINK_OPT }, \
78 {"module", required_argument, 0, 'P'}, \
79 {"proxy", required_argument, 0, 'x'}, \
80 {"redirection", required_argument, 0, REDIRECTION_OPT }, \
81 {"conn-timeout", required_argument, 0, CONN_TIMEOUT }, \
82 {"retry", required_argument, 0, RETRY_OPT }, \
83 {"retry-delay", required_argument, 0, RETRY_DELAY_OPT }, \
84 {"timeout", required_argument, 0, TIMEOUT_OPS }, \
85 {"trace", required_argument, 0, TRACE_OPTIONS }, \
86 {"infile", required_argument, 0, HAS_INPUT_FILE }, \
87 {"version", no_argument, 0, 'V'}
88
89 #define SECURITY_LONG_OPTIONS \
90 {"cert", required_argument, 0, 'E' }, \
91 {"capath", required_argument, 0, CAPATH_OPT }, \
92 {"key", required_argument, 0, X509_PRIVATE_KEY}, \
93 {"userlogin", required_argument, 0, USER_LOGIN}, \
94 {"userpass", required_argument, 0, USER_PASSWORD}, \
95 {"s3secretkey", required_argument, 0, S3_SECRET_KEY}, \
96 {"s3accesskey", required_argument, 0, S3_ACCESS_KEY}, \
97 {"s3region", required_argument, 0, S3_REGION}, \
98 {"s3token", required_argument, 0, S3_TOKEN}, \
99 {"azurekey", required_argument, 0, AZURE_KEY}, \
100 {"s3alternate", no_argument, 0, S3_ALTERNATE}, \
101 {"gcloud-creds", required_argument, 0, GCLOUD_CRED_PATH}, \
102 {"insecure", no_argument, 0, 'k' }, \
103 {"ostoken", required_argument, 0, OS_TOKEN}, \
104 {"osprojectid", required_argument, 0, OS_PROJECT_ID}, \
105 {"swiftaccount", required_argument, 0, SWIFT_ACCOUNT}
106
107 #define REQUEST_LONG_OPTIONS \
108 {"request", required_argument, 0, 'X' }, \
109 {"data", required_argument, 0, DATA_CONTENT}, \
110 {"verbose", no_argument, 0, 0 }
111
112 #define LISTING_LONG_OPTIONS \
113 {"s3-listing", required_argument, 0, S3_LISTING_MODE }, \
114 {"s3-maxkeys", required_argument, 0, S3_MAX_KEYS }, \
115 {"no-cap", required_argument, 0, DISABLE_LISTING_CAP}, \
116 {"long-list", no_argument, 0, 'l' }, \
117 {"swift-listing", required_argument, 0, SWIFT_LISTING_MODE}
118
119 #define GET_LONG_OPTIONS \
120 {"accepted-retry", required_argument, 0, ACCEPTED_RETRY}, \
121 {"accepted-retry-delay", required_argument, 0, ACCEPTED_RETRY_DELAY}
122
123 #define PUT_LONG_OPTIONS \
124 {"no-100-continue", no_argument, 0, NO_100_CONTINUE }
125
126 #define COPY_LONG_OPTIONS \
127 {"copy-mode", required_argument, 0, THIRD_PT_COPY_MODE }
128
129 OptParams::OptParams() :
130 params(),
131 vec_arg(),
132 debug(false),
133 s3_delete_per_request(20),
134 threadpool_size(10),
135 req_type(),
136 help_msg(),
137 cred_path(),
138 priv_key(),
139 cred_pass(),
140 output_file_path(),
141 input_file_path(),
142 userlogpasswd(),
143 userlogpasswd_through_cmdline(false),
144 req_content(),
145 aws_auth(),
146 aws_region(),
147 aws_token(),
148 aws_alternate(false),
149 os_token(),
150 os_project_id(),
151 swift_account(),
152 pres_flag(0),
153 shell_flag(0),
154 has_input_file(false),
155 no_cap(false)
156 {
157
158 }
159
160
161 static void option_abort(char** argv){
162 std::cerr << argv[0] <<", Error: Wrong argument format\n";
163 std::cerr << "Try '" << argv[0] <<" --help' for more information" << std::endl;
164 exit(-1);
165 }
166
167 static void display_version(){
168 std::cout << "Version: " << version() << std::endl;
169 std::cout << "Runtime curl version: " << backendRuntimeVersion() << std::endl;
170 std::cout << "Compiled against curl " << backendHeadersVersion() << std::endl;
171
172 exit(0);
173 }
174
175 template <typename T, typename Y, typename Z>
176 Y match_option(T begin, T end,
177 Y begin_res, Y end_res,
178 Z val, char** argv){
179 Y res = match_array(begin, end, begin_res, end_res, val);
180 if(res == end_res){
181 option_abort(argv);
182 }
183 return res;
184 }
185
186 static int set_header_field(const std::string & arg, OptParams & p, DavixError** err){
187 dav_size_t pos;
188 if( (pos = arg.find(':') ) == std::string::npos){
189 DavixError::setupError(err, scope_params, StatusCode::InvalidArgument, " Invalid header field argument");
190 return -1;
191 }
192
193 dav_size_t value_pos = pos+1;
194 if(arg.size() > pos+1 && arg[pos+1] == ' ') {
195 value_pos = pos+2;
196 }
197 p.params.addHeader(arg.substr(0,pos), arg.substr(value_pos));
198 return 0;
199 }
200
201 static void set_metalink_opt(RequestParams & params, const std::string & metalink_opt, char** argv){
202 const std::string str_opt[] = { "no" , "disable", "auto", "failover" };
203 const Davix::MetalinkMode::MetalinkMode mode_opt[] = { MetalinkMode::Disable, MetalinkMode::Disable, MetalinkMode::Auto, MetalinkMode::FailOver };
204 params.setMetalinkMode(*match_option(str_opt, str_opt+sizeof(str_opt)/sizeof(str_opt[0]),
205 mode_opt, mode_opt + sizeof(mode_opt)/sizeof(mode_opt[0]),
206 metalink_opt, argv));
207 }
208
209
210 static void set_redirection_opt(RequestParams & params, const std::string & redir_opt, char** argv){
211 const std::string str_opt[] = { "no" , "yes", "auto"};
212 const bool mode_opt[] = { false, true, true };
213 params.setTransparentRedirectionSupport(*match_option(str_opt, str_opt+sizeof(str_opt)/sizeof(str_opt[0]),
214 mode_opt, mode_opt + sizeof(mode_opt)/sizeof(mode_opt[0]),
215 redir_opt, argv));
216 }
217
218
219 static int parse_int(const std::string & opt, char** argv){
220 try{
221 return toType<int, std::string>()(opt);
222 }catch(...){
223 std::cerr << "Invalid option value " << opt << std::endl;
224 option_abort(argv);
225 }
226 return 0;
227 }
228
229 static struct timespec parse_timeout(const std::string & opt, char** argv){
230 int t = parse_int(opt, argv);
231
232 struct timespec timelapse;
233 timelapse.tv_sec =t;
234 timelapse.tv_nsec =0;
235 return timelapse;
236 }
237
238 void parse_davix_config(OptParams &p, std::string url) {
239 if(davix_config_apply("~/.davixrc", p, url)) return;
240 if(davix_config_apply("~/.netrc", p, url)) return;
241 }
242
243 int parse_davix_options_generic(const std::string &opt_filter,
244 const struct option* long_options,
245 int argc, char** argv, OptParams & p, DavixError** err){
246 int ret =0;
247 int option_index=0;
248
249 while( (ret = getopt_long(argc, argv, opt_filter.c_str(),
250 long_options, &option_index)) > 0){
251
252 switch(ret){
253 case DEBUG_OPT:
254 p.debug = true;
255 // max log level
256 setLogLevel(DAVIX_LOG_TRACE);
257 setLogScope(DAVIX_LOG_SCOPE_ALL ^ DAVIX_LOG_XML);
258 break;
259 case 'E':
260 p.cred_path = SanitiseTildedPath(optarg);
261 break;
262 case 'k':
263 p.params.setSSLCAcheck(false);
264 break;
265 case 'H':
266 if( set_header_field(optarg, p, err) <0)
267 return -1;
268 break;
269 case S3_LISTING_MODE:
270 {
271 if(std::string(optarg).compare("flat")==0)
272 p.params.setS3ListingMode(S3ListingMode::Flat);
273 else if(std::string(optarg).compare("hierarchical")==0)
274 p.params.setS3ListingMode(S3ListingMode::Hierarchical);
275 else if(std::string(optarg).compare("semi")==0)
276 p.params.setS3ListingMode(S3ListingMode::SemiHierarchical);
277 break;
278 }
279 case S3_MAX_KEYS:
280 p.params.setS3MaxKey(atoi(optarg));
281 break;
282 case SWIFT_LISTING_MODE:
283 {
284 if(std::string(optarg).compare("hierarchical")==0)
285 p.params.setSwiftListingMode(SwiftListingMode::Hierarchical);
286 else if(std::string(optarg).compare("semi")==0)
287 p.params.setSwiftListingMode(SwiftListingMode::SemiHierarchical);
288 break;
289 }
290 case CAPATH_OPT:
291 p.params.addCertificateAuthorityPath(optarg);
292 break;
293 case USER_LOGIN:
294 p.userlogpasswd.first = optarg;
295 p.userlogpasswd_through_cmdline = true;
296 strncpy(optarg, "", strlen(optarg));
297 break;
298 case X509_PRIVATE_KEY:
299 p.priv_key = SanitiseTildedPath(optarg).c_str();
300 break;
301 case USER_PASSWORD:
302 p.userlogpasswd.second = optarg;
303 p.userlogpasswd_through_cmdline = true;
304 strncpy(optarg, "", strlen(optarg));
305 break;
306 case DATA_CONTENT:
307 p.req_content = optarg;
308 break;
309 case RETRY_OPT:
310 p.params.setOperationRetry( parse_int(optarg, argv));
311 break;
312 case RETRY_DELAY_OPT:
313 p.params.setOperationRetryDelay( parse_int(optarg, argv));
314 break;
315 case S3_ACCESS_KEY:
316 p.aws_auth.second = optarg;
317 strncpy(optarg, "", strlen(optarg));
318 break;
319 case S3_SECRET_KEY:
320 p.aws_auth.first = optarg;
321 strncpy(optarg, "", strlen(optarg));
322 break;
323 case S3_REGION:
324 p.aws_region = optarg;
325 break;
326 case S3_TOKEN:
327 p.aws_token = optarg;
328 strncpy(optarg, "", strlen(optarg));
329 break;
330 case S3_ALTERNATE:
331 p.aws_alternate = true;
332 break;
333 case AZURE_KEY:
334 p.azure_key = optarg;
335 strncpy(optarg, "", strlen(optarg));
336 break;
337 case GCLOUD_CRED_PATH:
338 p.gcloud_creds_path = SanitiseTildedPath(optarg).c_str();
339 break;
340 case OS_TOKEN:
341 p.os_token = optarg;
342 strncpy(optarg, "", strlen(optarg));
343 break;
344 case OS_PROJECT_ID:
345 p.os_project_id = optarg;
346 strncpy(optarg, "", strlen(optarg));
347 break;
348 case SWIFT_ACCOUNT:
349 p.swift_account = optarg;
350 strncpy(optarg, "", strlen(optarg));
351 break;
352 case 'l':
353 p.pres_flag |= LONG_LISTING_FLAG;
354 break;
355 case 'o':
356 p.output_file_path = optarg;
357 break;
358 case 'P':
359 p.modules_list = StrUtil::tokenSplit(std::string(optarg), ",");
360 break;
361 case 'V':
362 display_version();
363 return 1;
364 case TRACE_OPTIONS:
365 {
366 p.debug = true;
367 setLogScope(0);
368 setLogScope(std::string(optarg));
369 setLogLevel(DAVIX_LOG_TRACE);
370 }
371 break;
372 case 'x':
373 p.params.setProxyServer(std::string(optarg, 0, 2048));
374 break;
375 case 'X':
376 p.req_type = std::string(optarg, 0, 255);
377 break;
378 case METALINK_OPT:
379 set_metalink_opt(p.params, std::string(optarg), argv);
380 break;
381 case REDIRECTION_OPT:
382 set_redirection_opt(p.params, std::string(optarg), argv);
383 break;
384 case CONN_TIMEOUT:{
385 struct timespec s =parse_timeout(std::string(optarg), argv);
386 p.params.setConnectionTimeout(&s);
387 }
388 break;
389 case TIMEOUT_OPS:{
390 struct timespec s =parse_timeout(std::string(optarg), argv);
391 p.params.setOperationTimeout(&s);
392 break;
393 }
394 case HAS_INPUT_FILE:{
395 p.has_input_file = true;
396 p.input_file_path = optarg;
397 }
398 break;
399 case THIRD_PT_COPY_MODE:{
400 if(std::string(optarg).compare("pull")==0)
401 p.params.setCopyMode(CopyMode::Pull);
402 else if(std::string(optarg).compare("push")==0)
403 p.params.setCopyMode(CopyMode::Push);
404 break;
405 }
406 case DISABLE_LISTING_CAP:{
407 p.no_cap = true;
408 break;
409 }
410 case 'r':{
411 p.params.setRecursiveMode(true);
412 p.threadpool_size = atoi(optarg);
413 break;
414 }
415 case 'n':{
416 p.s3_delete_per_request = (atoi(optarg));
417 break;
418 }
419 case NO_100_CONTINUE:{
420 p.params.set100ContinueSupport(false);
421 break;
422 }
423 case ACCEPTED_RETRY:
424 std::cout << "in accepted retry" << std::endl;
425 p.params.setAcceptedRetry(atoi(optarg));
426 break;
427 case ACCEPTED_RETRY_DELAY:
428 p.params.setAcceptedRetryDelay(atoi(optarg));
429 break;
430 case '?':
431 std::cout << p.help_msg;
432 exit(1);
433 break;
434 default:
435 option_abort(argv);
436 }
437 }
438
439
440 ret =-1;
441 for(int i = optind; i < argc; ++i){
442 p.vec_arg.push_back(argv[i]);
443 ret =0;
444 }
445
446 if(ret != 0){
447 option_abort(argv);
448 }
449
450 return ret;
451 }
452
453
454
455 int parse_davix_options(int argc, char** argv, OptParams & p, DavixError** err){
456 const std::string arg_tool_main= "P:x:H:E:X:o:kVr:";
457 const struct option long_options[] = {
458 COMMON_LONG_OPTIONS,
459 SECURITY_LONG_OPTIONS,
460 REQUEST_LONG_OPTIONS,
461 {0, 0, 0, 0 }
462 };
463
464 if( parse_davix_options_generic(arg_tool_main, long_options,
465 argc, argv,
466 p, err) < 0) {
467 return -1;
468 }
469
470 parse_davix_config(p, p.vec_arg[0]);
471 return 0;
472 }
473
474
475 int parse_davix_ls_options(int argc, char** argv, OptParams & p, DavixError** err){
476 const std::string arg_tool_main= "P:x:H:E:vkVlfr:";
477 const struct option long_options[] = {
478 COMMON_LONG_OPTIONS,
479 SECURITY_LONG_OPTIONS,
480 LISTING_LONG_OPTIONS,
481 {0, 0, 0, 0 }
482 };
483
484 if( parse_davix_options_generic(arg_tool_main, long_options,
485 argc, argv,
486 p, err) <0
487 || p.vec_arg.size() != 1){
488 option_abort(argv);
489 return -1;
490 }
491 parse_davix_config(p, p.vec_arg[0]);
492 return 0;
493 }
494
495
496 int parse_davix_get_options(int argc, char** argv, OptParams & p, DavixError** err){
497 const std::string arg_tool_main= "P:x:H:E:o:OvkVr:";
498 const struct option long_options[] = {
499 COMMON_LONG_OPTIONS,
500 SECURITY_LONG_OPTIONS,
501 GET_LONG_OPTIONS,
502 {0, 0, 0, 0 }
503 };
504
505 if( parse_davix_options_generic(arg_tool_main, long_options,
506 argc, argv,
507 p, err) < 0
508 || p.vec_arg.size() > 2){
509 option_abort(argv);
510 return -1;
511 }
512
513 if(p.vec_arg.size() == 2){
514 p.output_file_path = p.vec_arg[1];
515 }
516 parse_davix_config(p, p.vec_arg[0]);
517 return 0;
518 }
519
520 int parse_davix_put_options(int argc, char** argv, OptParams & p, DavixError** err){
521 const std::string arg_tool_main= "P:x:H:E:o:vkVr:";
522 const struct option long_options[] = {
523 COMMON_LONG_OPTIONS,
524 SECURITY_LONG_OPTIONS,
525 PUT_LONG_OPTIONS,
526 {0, 0, 0, 0 }
527 };
528
529 if( parse_davix_options_generic(arg_tool_main,
530 long_options,
531 argc,
532 argv,
533 p, err) < 0
534 || p.vec_arg.size() != 2){
535
536 option_abort(argv);
537 return -1;
538 }
539 p.input_file_path = p.vec_arg[0];
540 parse_davix_config(p, p.vec_arg[1]);
541 return 0;
542 }
543
544 int parse_davix_copy_options(int argc, char** argv, OptParams & p, DavixError** err){
545 const std::string arg_tool_main= "P:x:H:E:X:o:kV";
546 const struct option long_options[] = {
547 COMMON_LONG_OPTIONS,
548 SECURITY_LONG_OPTIONS,
549 REQUEST_LONG_OPTIONS,
550 COPY_LONG_OPTIONS,
551 {0, 0, 0, 0 }
552 };
553
554 if(parse_davix_options_generic(arg_tool_main, long_options,
555 argc, argv,
556 p, err) < 0) {
557 return -1;
558 }
559
560 parse_davix_config(p, p.vec_arg[0]);
561 return 0;
562 }
563
564 int parse_davix_rm_options(int argc, char** argv, OptParams & p, DavixError** err){
565 const std::string arg_tool_main= "P:x:H:E:X:o:kVr:n:";
566 const struct option long_options[] = {
567 COMMON_LONG_OPTIONS,
568 SECURITY_LONG_OPTIONS,
569 REQUEST_LONG_OPTIONS,
570 {0, 0, 0, 0 }
571 };
572
573 if( parse_davix_options_generic(arg_tool_main, long_options,
574 argc, argv,
575 p, err) < 0) {
576 return -1;
577 }
578
579 parse_davix_config(p, p.vec_arg[0]);
580 return 0;
581 }
582
583 std::string get_common_options(){
584 return " Common Options:\n"
585 "\t--conn-timeout TIME: Connection timeout in seconds. default: 30\n"
586 "\t--retry NUMBER: Number of retry attempts in case of an operation failure. default: 3\n"
587 "\t--retry-delay TIME: Number of seconds to wait between retry attempts. default: 0\n"
588 "\t--debug: Debug mode\n"
589 "\t--header, -H: Add a header field to the request\n"
590 "\t--help, -h: Display this help message\n"
591 "\t--metalink OPT: Metalink support. value=failover|no. default=failover) \n"
592 "\t--module, -P NAME: Load a plugin or profile by name\n"
593 "\t--proxy, -x URL: SOCKS5 proxy server URL. (Ex: socks5://login:[email protected])\n"
594 "\t--redirection OPT: Transparent redirection support. value=yes|no. default=yes)\n"
595 "\t--timeout TIME: Global timeout for the operation in seconds. default: infinite\n"
596 "\t--trace: Specify one or more scopes to trace. (Ex: --trace log level(optional),header,file)\n"
597 "\t--version, -V: Display version\n"
598 " Security Options:\n"
599 "\t--capath CA_PATH: Add an additional certificate authority directory\n"
600 "\t--cert, -E CRED_PATH: Client Certificate in PEM format\n"
601 "\t--key KEY_PATH: Private key in PEM format\n"
602 "\t--insecure, -k: Disable SSL credential checks\n"
603 "\t--userlogin: User login for login/password authentication\n"
604 "\t--userpass: User password for login/password authentication\n"
605 "\t--s3secretkey SEC_KEY: S3 authentication: secret key\n"
606 "\t--s3accesskey ACC_KEY: S3 authentication: access key\n"
607 "\t--s3region REGION: S3 region (optional - if passed, will authenticate using a v4 signature instead of v2)\n"
608 "\t--s3token TOKEN: S3 security token - used along with STS temporary credentials\n"
609 "\t--s3alternate: Pass this flag if you're using a path-based S3 URL\n"
610 "\t A path-based URL contains the bucket name in the path, ie https://s3-someregion.amazonaws.com/mybucket/file\n"
611 "\t--azurekey AZURE_KEY: Azure authentication secret key\n"
612 "\t--gcloud-creds PATH: Path to gcloud json credentials\n"
613 "\t--ostoken TOKEN: Swift authentication: Openstack token\n"
614 "\t--osprojectid: Swift authentication: Openstack project ID\n"
615 "\t--swiftaccount: Alternative Swift authentication: Swift account\n"
616 ;
617 }
618
619
620
621 std::string get_base_description_options(){
622 return "[OPTIONS ...] <url>\n"
623 ;
624 }
625
626 std::string get_get_description_options(){
627 return "[OPTIONS ...] <url> [local_file]\n"
628 ;
629 }
630
631
632
633 std::string get_put_description_options(){
634 return "[OPTIONS ...] <local_file> <remote_file_url> \n"
635 ;
636 }
637
638
639 std::string get_copy_description_options(){
640 return "[OPTIONS ...] <src_url> <dst_url>\n"
641 ;
642 }
643
644 }
645
646 }