Debian Codegraph @

src/neon/neonrequest.cpp

raw
1 /*
2 * This File is part of Davix, The IO library for HTTP based protocols
3 * Copyright (C) CERN 2013
4 * Author: Adrien Devresse <[email protected]>
5 *
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2.1 of the License, or (at your option) any later version.
10 *
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, write to the Free Software
18 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19 *
20 */
21
22 #include <davix_internal.hpp>
23 #include "neonrequest.hpp"
24
25 #include <core/ContentProvider.hpp>
26 #include <utils/davix_logger_internal.hpp>
27 #include <utils/davix_gcloud_utils.hpp>
28 #include <ne_redirect.h>
29 #include <ne_request.h>
30 #include <davix_context_internal.hpp>
31 #include <neon/neonsession.hpp>
32 #include <fileops/fileutils.hpp>
33 #include <fileops/AzureIO.hpp>
34 #include <fileops/S3IO.hpp>
35 #include <core/RedirectionResolver.hpp>
36 #include <utils/CompatibilityHacks.hpp>
37 #include <backend/SessionFactory.hpp>
38 #include <curl/StandaloneCurlRequest.hpp>
39
40 #include "../backend/StandaloneNeonRequest.hpp"
41
42 #define DBG(message) std::cerr << __FILE__ << ":" << __LINE__ << " -- " << #message << " = " << message << std::endl
43
44 namespace Davix {
45
46 void configureRequestParamsProto(const Uri &uri, RequestParams &params){
47 if(params.getProtocol() == RequestProtocol::Auto){
48 const std::string & proto = uri.getProtocol();
49 if( proto.compare(0,2,"s3") ==0){
50 params.setProtocol(RequestProtocol::AwsS3);
51 }else if ( proto.compare(0, 3,"dav") ==0){
52 params.setProtocol(RequestProtocol::Webdav);
53 }else if ( proto.compare(0, 6, "gcloud") ==0) {
54 params.setProtocol(RequestProtocol::Gcloud);
55 }else if (proto.compare(0, 3, "cs3")==0){
56 params.setProtocol(RequestProtocol::CS3);
57 }
58 }
59 }
60
61 void neon_generic_error_mapper(int ne_status, StatusCode::Code & code, std::string & str, const std::string &wwwAuth){
62 switch(ne_status){
63 case NE_OK:
64 code = StatusCode::OK;
65 str = "Status Ok";
66 break;
67 case NE_LOOKUP:
68 code = StatusCode::NameResolutionFailure;
69 str = "Domain name resolution failed";
70 break;
71 case NE_AUTH:
72 code = StatusCode::AuthenticationError;
73 str = "Authentication failed on server";
74 break;
75 case NE_PROXYAUTH:
76 code = StatusCode::AuthenticationError;
77 str = "Authentication failed on proxy";
78 break;
79 case NE_CONNECT:
80 code = StatusCode::ConnectionProblem;
81 str = "Could not connect to server";
82 break;
83 case NE_TIMEOUT:
84 code = StatusCode::ConnectionTimeout;
85 str = "Connection timed out";
86 break;
87 case NE_FAILED:
88 code = StatusCode::SessionCreationError;
89 str = "The precondition failed";
90 break;
91 case NE_RETRY:
92 code = StatusCode::RedirectionNeeded;
93 str = "Retry Request";
94 break;
95 default:
96 code = StatusCode::UnknownError;
97 str = "Unknown Error from libneon";
98 }
99
100 if(!wwwAuth.empty()) {
101 str += "(WWW-Authenticate: ";
102 str += wwwAuth;
103 str += ")";
104 }
105 }
106
107
108 NeonRequest::NeonRequest(const BoundHooks &hooks, Context& context, const Uri & uri_req) :
109 BackendRequest(context, uri_req),
110 _bound_hooks(hooks),
111 _number_try(0),
112 _redirects(0),
113 _total_read_size(0),
114 _headers_configured(false),
115 _accepted_202_retries(0) {
116 }
117
118
119 NeonRequest::~NeonRequest(){
120 // safe destruction of the request
121 freeRequest();
122 }
123
124 void NeonRequest::cancelSessionReuse(){
125 // if session registered
126 // cancel any re-use of the session
127 //
128 if(_standalone_req) {
129 DAVIX_SLOG(DAVIX_LOG_DEBUG, DAVIX_LOG_HTTP, "Connection problem: eradicate session");
130 _standalone_req->doNotReuseSession();
131 }
132 }
133
134 //------------------------------------------------------------------------------
135 // Prepare URI & params
136 //------------------------------------------------------------------------------
137 void NeonRequest::prepareUriParams() {
138 // reconfigure protos
139 configureRequestParamsProto(*_current, _params);
140
141 // configure S3 params if needed
142 if(_params.getProtocol() == RequestProtocol::AwsS3)
143 configureS3params();
144
145 // configure azure params if needed
146 if(_params.getProtocol() == RequestProtocol::Azure)
147 configureAzureParams();
148
149 // configure gcloud params if needed
150 if(_params.getProtocol() == RequestProtocol::Gcloud)
151 configureGcloudParams();
152
153 // configure swift params if needed
154 if(_params.getProtocol() == RequestProtocol::Swift)
155 configureSwiftParams();
156 }
157
158 //------------------------------------------------------------------------------
159 // Check redirect cache
160 //------------------------------------------------------------------------------
161 void NeonRequest::checkRedirectCache() {
162 std::shared_ptr<Uri> redir_url;
163 if(this->_params.getTransparentRedirectionSupport()) {
164 redir_url = ContextExplorer::RedirectionResolverFromContext(_context).redirectionResolve(_request_type, *_current);
165 }
166
167 // performing an operation which could change the PFN? Clear all cache entries for selected URL
168 if(_request_type == "DELETE" || _request_type == "MOVE") {
169 ContextExplorer::RedirectionResolverFromContext(_context).redirectionClean(*_current.get());
170 }
171
172 if(redir_url.get()) {
173 _current = redir_url;
174 }
175 }
176
177 void NeonRequest::configureHeaders() {
178 // add custom user headers, but make sure they're only added once
179 // in case of a redirect
180
181 if(!_headers_configured) {
182 std::copy(_params.getHeaders().begin(), _params.getHeaders().end(), std::back_inserter(_headers_field));
183 _headers_configured = true;
184 }
185 }
186
187 //------------------------------------------------------------------------------
188 // Should we use libcurl?
189 //------------------------------------------------------------------------------
190 static bool useLibcurl() {
191 const char *opt = getenv("DAVIX_USE_LIBCURL");
192 if(opt) {
193 if(opt[0] == '1' || opt[0] == 'Y' || opt[0] == 'y') {
194 return true;
195 }
196
197 if(opt[0] == '0' || opt[0] == 'N' || opt[0] == 'n') {
198 return false;
199 }
200 }
201
202 #ifdef LIBCURL_BACKEND_BY_DEFAULT
203 return true;
204 #else
205 return false;
206 #endif
207 }
208
209 //------------------------------------------------------------------------------
210 // Initialize standalone request
211 //------------------------------------------------------------------------------
212 void NeonRequest::initStandaloneRequest() {
213 if(useLibcurl()) {
214 CurlSessionFactory& factory = ContextExplorer::SessionFactoryFromContext(getContext()).getCurl();
215 _standalone_req.reset(new StandaloneCurlRequest(
216 factory,
217 true,
218 _bound_hooks,
219 * (_current.get()),
220 _request_type,
221 _params,
222 _headers_field,
223 _req_flag,
224 _content_provider,
225 _deadline
226 ));
227 }
228 else {
229 NEONSessionFactory& factory = ContextExplorer::SessionFactoryFromContext(getContext()).getNeon();
230 _standalone_req.reset(new StandaloneNeonRequest(
231 factory,
232 true,
233 _bound_hooks,
234 * (_current.get()),
235 _request_type,
236 _params,
237 _headers_field,
238 _req_flag,
239 _content_provider,
240 _deadline
241 ));
242 }
243 }
244
245 //------------------------------------------------------------------------------
246 // Initialize and configure _standalone_req
247 //------------------------------------------------------------------------------
248 void NeonRequest::createBackendRequest() {
249 configureHeaders();
250 checkRedirectCache();
251 prepareUriParams();
252
253 //--------------------------------------------------------------------------
254 // Doing a PUT to Azure? Detect if this is happening, and engage
255 // Azure-specific upload machinery.
256 //--------------------------------------------------------------------------
257 if(_content_provider && CompatibilityHacks::azureChunkedUpload(_request_type, *_current.get(), _context, _params, *_content_provider,
258 &_early_termination_error)) {
259 _early_termination = true;
260 return;
261 }
262
263 // setup timeout
264 setupDeadlineIfUnset();
265
266 // setup standalone req object
267 initStandaloneRequest();
268
269 // configure connexion parameters for PUT request
270 if( (_req_flag & RequestFlag::SupportContinue100) == true) {
271 _standalone_req->doNotReuseSession();
272 }
273 }
274
275 int NeonRequest::processRedirection(DavixError **err){
276 if(redirectRequest(err) <0){ // accept redirection
277 return -1;
278 }
279 return NE_RETRY;
280 }
281
282 int NeonRequest::startRequest(DavixError **err){
283 createBackendRequest();
284 return negotiateRequest(err);
285 }
286
287 int NeonRequest::negotiateRequest(DavixError** err){
288 const int auth_retry_limit = _params.getOperationRetry();
289 int code, end_status = NE_RETRY;
290 _total_read_size = 0;
291
292
293 DAVIX_SCOPE_TRACE(DAVIX_LOG_HTTP, negoreq);
294
295 // check timeout
296 if(checkTimeout(err) == true)
297 return -1;
298
299 while(end_status == NE_RETRY && _number_try <= auth_retry_limit) {
300 DAVIX_SLOG(DAVIX_LOG_TRACE, DAVIX_LOG_HTTP, "NEON start internal request");
301
302 Status st = _standalone_req->startRequest();
303
304 if(!st.ok()) {
305 _number_try++;
306 if(_number_try <= auth_retry_limit) {
307 DAVIX_SLOG(DAVIX_LOG_VERBOSE, DAVIX_LOG_HTTP, "Connection problem, retry");
308 requestCleanup();
309 return startRequest(err);
310 }
311
312 st.toDavixError(err);
313
314 cancelSessionReuse();
315 ContextExplorer::RedirectionResolverFromContext(_context).redirectionClean(_request_type, *_orig);
316 return -1;
317 }
318
319 code = getRequestCode();
320 switch(code){
321 case 202:
322 // azure will reply with 202 when deleting files :(
323 // Only activate retries on GET.
324 if(_request_type != "GET") goto default_label;
325
326 _accepted_202_retries++;
327 if(_accepted_202_retries <= _params.getAcceptedRetry()) {
328 std::cerr << "DAVIX: Received 202-Accepted, sleeping for " <<
329 _params.getAcceptedRetryDelay() << " seconds before retrying. (attempt " <<
330 _accepted_202_retries << " out of " << _params.getAcceptedRetry() << ")" << std::endl;
331 sleep(_params.getAcceptedRetryDelay());
332 endRequest(NULL);
333 return startRequest(err);
334 }
335 goto default_label;
336 case 301:
337 case 302:
338 case 303:
339 case 307:
340
341 if(_content_provider && CompatibilityHacks::dynafedAssistedS3Upload(*this, *_current.get(), _context, _params, *_content_provider,
342 &_early_termination_error)) {
343
344 // Dynafed mechanism was engaged, end request
345 requestCleanup();
346 DavixError::clearError(err);
347 endRequest(NULL);
348
349 _early_termination = true;
350 if(!_early_termination_error) return 0;
351 return -1;
352 }
353
354 if(!this->_params.getTransparentRedirectionSupport()) {
355 // We don't follow redirects, give response as-is.
356 return NE_OK;
357 }
358
359 _redirects++;
360 if(_redirects > NEON_REDIRECT_LIMIT) {
361 httpcodeToDavixError(code, davix_scope_http_request(), "Too many redirects", err);
362 return -1;
363 }
364
365 if( (end_status = processRedirection(err)) <0){
366 return -1;
367 }
368
369 if(_early_termination) {
370 end_status = 0;
371 break;
372 }
373
374 _number_try--;
375
376
377 break;
378 case 401: // authentication requested, do retry
379 case 403:
380 clearAnswerContent();
381
382 _number_try++;
383 if (_number_try <= auth_retry_limit && requestCleanup()){
384 DavixError::clearError(err);
385 endRequest(NULL);
386 return startRequest(err);
387 }
388
389 httpcodeToDavixError(code, davix_scope_http_request(), "", err);
390 return -1;
391 default:
392 default_label:
393 if(code >= 400) {
394 httpcodeToDavixError(code, davix_scope_http_request(), "", err);
395 }
396 return 0;
397
398 }
399 _number_try++;
400 }
401
402 if(end_status == NE_RETRY) {
403 DavixError::setupError(err,davix_scope_http_request(), StatusCode::AuthenticationError,
404 "Authentication error, reached maximum number of attempts");
405 return -2;
406 }
407
408 return 0;
409 }
410
411
412 // mark the session as dirty for destruction, clean any cached redirection due to error
413 // detect if the request comes from a cached session or have been redirected
414 // if it is the case, return true
415 //
416 bool NeonRequest::requestCleanup(){
417 // cleanup redirection
418 ContextExplorer::RedirectionResolverFromContext(_context).redirectionClean(_request_type, *_orig);
419
420 // disable recycling
421 // server supporting broken pipelining will trigger if reused
422 if(_standalone_req) {
423 _standalone_req->doNotReuseSession();
424 }
425
426 // check if we had a redirection
427 // if it's the case redirection can be expired, then
428 if(_current != _orig || _standalone_req->isRecycledSession()){
429 DAVIX_SLOG(DAVIX_LOG_DEBUG, DAVIX_LOG_HTTP, " -> Error when using reycling of session/redirect : cancel and try again");
430 // disable session reuse for this request, avoid picking up a session already expired
431 _params.setKeepAlive(false);
432 _current = _orig;
433 return true;
434 }
435 return false;
436 }
437
438 int NeonRequest::redirectRequest(DavixError **err) {
439 Uri location;
440 Status st = _standalone_req->obtainRedirectedLocation(location);
441
442 if(!st.ok()) {
443 st.toDavixError(err);
444 return -1;
445 }
446
447 // setup new path & session target
448 std::shared_ptr<Uri> old_uri = _current;
449 _current= std::shared_ptr<Uri>(new Uri(location));
450 ContextExplorer::RedirectionResolverFromContext(_context).addRedirection(_request_type, *old_uri, _current);
451
452 // recycle old request and session
453 freeRequest();
454
455 // create a new couple of session + request
456 createBackendRequest();
457 return 0;
458 }
459
460 int NeonRequest::executeRequest(DavixError** err){
461 dav_ssize_t read_status=1, total_read=0;
462 _vec.clear();
463
464 DAVIX_SCOPE_TRACE(DAVIX_LOG_HTTP, execReq);
465
466 if( startRequest(err) < 0){
467 return -1;
468 }
469
470 if(getAnswerSize() > 0)
471 _vec.reserve(std::min<size_t>(getAnswerSize(), 4194304));
472
473 while(read_status > 0){
474 DAVIX_SLOG(DAVIX_LOG_DEBUG, DAVIX_LOG_HTTP, "NEON Read data flow");
475 size_t s = _vec.size();
476 _vec.resize(s + NEON_BUFFER_SIZE);
477 read_status= readBlock(&(_vec[s]), NEON_BUFFER_SIZE, err);
478 if( read_status >= 0){
479 if(read_status != NEON_BUFFER_SIZE){
480 _vec.resize(s + read_status);
481 }
482 total_read += read_status;
483 }
484 }
485
486 if(read_status < 0){
487 if(err && *err == NULL){
488 createError(read_status, err);
489 }
490 return -1;
491 }
492 _vec.push_back('\0');
493
494 if(_ans_size < 0){
495 _ans_size = total_read;
496 }
497
498 if( endRequest(err) < 0){
499 return -1;
500 }
501
502 return 0;
503 }
504
505 int NeonRequest::beginRequest(DavixError** err){
506 if(_standalone_req) {
507 DavixError::setupError(err, davix_scope_http_request(), StatusCode::AlreadyRunning, "Http request already started, Error");
508 return -1;
509 }
510
511 int ret = -1;
512 _vec.clear();
513 if( (ret= startRequest(err)) < 0)
514 return -1;
515
516 return ret;
517 }
518
519 dav_ssize_t NeonRequest::readBlock(char* buffer, dav_size_t max_size, DavixError** err){
520 dav_ssize_t read_status=-1;
521
522 if(!_standalone_req) {
523 DavixError::setupError(err, davix_scope_http_request(), StatusCode::AlreadyRunning, "No request started");
524 return -1;
525 }
526
527 if(max_size ==0)
528 return 0;
529
530 // check timeout
531 if(checkTimeout(err) == true)
532 return -1;
533
534 // take from line buffer
535 if(_vec_line.size() > 0){
536 if( _vec_line.size() >= max_size){
537 std::copy(_vec_line.begin(), _vec_line.begin() + max_size, buffer);
538 _vec_line.erase(_vec_line.begin(), _vec_line.begin() + max_size);
539 DAVIX_SLOG(DAVIX_LOG_TRACE, DAVIX_LOG_HTTP, "NeonRequest::readBlock read {} bytes (from buffer)", max_size);
540 return max_size;
541 }else{
542 const dav_ssize_t n_bytes = _vec_line.size();
543 std::copy(_vec_line.begin(), _vec_line.end(), buffer);
544 _vec_line.clear();
545 read_status = readBlock(buffer + n_bytes, max_size -n_bytes, err);
546 const dav_ssize_t ret_value = (read_status >= 0)?(read_status+n_bytes):(-1);
547 DAVIX_SLOG(DAVIX_LOG_TRACE, DAVIX_LOG_HTTP, "NeonRequest::readBlock read {} bytes(from partially)", ret_value);
548 return ret_value;
549 }
550 }
551
552 if(_standalone_req) {
553 Status st;
554 dav_ssize_t retval = _standalone_req->readBlock(buffer, max_size, st);
555 if(!st.ok()) {
556 st.toDavixError(err);
557 }
558 return retval;
559 }
560
561 DAVIX_SLOG(DAVIX_LOG_TRACE, DAVIX_LOG_HTTP, "NeonRequest::readBlock read {} bytes", read_status);
562
563 _total_read_size += read_status;
564 if(_params.getTransferMonitorCb()){
565 dav_ssize_t final_size = getAnswerSize();
566 _params.getTransferMonitorCb()(*_current, Transfer::Read, _total_read_size, ((final_size> 0)?(final_size):(0)));
567 }
568 return read_status;
569 }
570
571 int NeonRequest::endRequest(DavixError** err){
572 if(!_standalone_req) {
573 DavixError::setupError(err, davix_scope_http_request(), StatusCode::InvalidArgument, "Request not started");
574 return -1;
575 }
576
577 Status st = _standalone_req->endRequest();
578
579 if(!st.ok()) {
580 st.toDavixError(err);
581 }
582
583 return st.okAsInt();
584 }
585
586 //------------------------------------------------------------------------------
587 // Get response status.
588 //------------------------------------------------------------------------------
589 int NeonRequest::getRequestCode() {
590 if(_early_termination) {
591 if(!_early_termination_error) return 200;
592 return _early_termination_error->getStatus();
593 }
594
595 if(_standalone_req) {
596 return _standalone_req->getStatusCode();
597 }
598
599 return 0;
600 }
601
602 bool NeonRequest::getAnswerHeader(const std::string &header_name, std::string &value) const {
603 if(_standalone_req) {
604 return _standalone_req->getAnswerHeader(header_name, value);
605 }
606
607 return false;
608 }
609
610 size_t NeonRequest::getAnswerHeaders( HeaderVec & vec_headers) const {
611 if(_standalone_req) {
612 return _standalone_req->getAnswerHeaders(vec_headers);
613 }
614
615 return vec_headers.size();
616 }
617
618 void NeonRequest::freeRequest(){
619 DavixError::clearError(&_early_termination_error);
620 _standalone_req.reset();
621 }
622
623 void NeonRequest::createError(int ne_status, DavixError **err){
624 StatusCode::Code code;
625 std::string str;
626 std::string wwwAuth;
627 _standalone_req->getAnswerHeader("WWW-Authenticate", wwwAuth);
628
629 switch(ne_status){
630 case NE_ERROR:
631 {
632 str = "(" + _standalone_req->getName() + "): " + _standalone_req->getSessionError();
633 if (str.find("SSL handshake failed") == std::string::npos)
634 code = StatusCode::ConnectionProblem;
635 else
636 code = StatusCode::SSLError;
637 }
638 break;
639 case NE_TIMEOUT:
640 {
641 // check if redirection occurred, if redirection occurred
642 // report TimeoutRedirectionError, to allow error recovery
643 if(_current != _orig){
644 code = StatusCode::TimeoutRedirectionError;
645 str = "Connection Timeout during redirection on ";
646 str+= _current->getString();
647 break;
648 }
649 neon_generic_error_mapper(ne_status, code, str, wwwAuth);
650 }
651 break;
652
653 default:
654 neon_generic_error_mapper(ne_status, code, str, wwwAuth);
655 break;
656 }
657 DavixError::setupError(err, davix_scope_http_request(), code, str);
658 }
659
660
661 } // namespace Davix