Debian Codegraph @

src/modules/copy/copy.cpp

raw
1 /*
2 * This File is part of Davix, The IO library for HTTP based protocols
3 * Copyright (C) CERN 2013
4 * Author: Adrien Devresse <[email protected]>
5 *
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2.1 of the License, or (at your option) any later version.
10 *
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, write to the Free Software
18 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19 *
20 */
21
22 #include <cstdio>
23 #include <cstring>
24 #include <davix.hpp>
25 #include <sstream>
26 #include <unistd.h>
27 #include <list>
28 #include <iterator>
29 #include "copy_internal.hpp"
30 #include "delegation/delegation.hpp"
31 #include <utils/davix_logger_internal.hpp>
32 #include <utils/davix_gcloud_utils.hpp>
33 #include <utils/stringutils.hpp>
34
35 using namespace Davix;
36
37 const std::string COPY_SCOPE = "Davix::HttpThirdPartyCopy";
38
39 // Gets the real location of uri respect to ref
40 // uri can be absolute or relative to ref
41 static std::string _full_url(const std::string ref,
42 const std::string& uri)
43 {
44 std::string final;
45
46 if (uri.find("://") != std::string::npos) {
47 final = uri;
48 }
49 else if (uri[0] == '/') {
50 size_t colon = ref.find(':');
51 size_t slash = std::string::npos;
52 if (colon != std::string::npos)
53 slash = ref.find('/', colon + 3);
54 if (slash != std::string::npos) {
55 std::string base = ref.substr(0, slash);
56 final = base + uri;
57 }
58 }
59 else {
60 final = ref + uri;
61 }
62
63 return final;
64 }
65
66
67
68 // Same as _full_url, but it makes sure
69 // that the destination is https
70 static std::string _full_delegation_endpoint(const std::string& ref,
71 const std::string& uri, DavixError** err)
72 {
73 std::string final = _full_url(ref, uri);
74 if (final.substr(7).compare("http://") == 0) {
75 DavixError::setupError(err, COPY_SCOPE, StatusCode::OperationNonSupported,
76 std::string("Plain http can not be used for delegation: ") + uri);
77 final.clear();
78 }
79 return final;
80 }
81
82 static std::vector<std::string> parseXDelegateTo(const std::string& ref,
83 const std::string& uris, DavixError** err) {
84
85 std::vector<std::string> endpoints = StrUtil::tokenSplit(uris, " ");
86 for(size_t i = 0; i < endpoints.size(); i++) {
87 endpoints[i] = _full_delegation_endpoint(ref, endpoints[i], err);
88 }
89 return endpoints;
90 }
91
92 DavixCopy::DavixCopy(Context &c, const RequestParams *params): d_ptr(NULL)
93 {
94 d_ptr = new DavixCopyInternal(c, params);
95 }
96
97
98
99 DavixCopy::~DavixCopy()
100 {
101 delete d_ptr;
102 }
103
104 void DavixCopy::copy(const Uri &source, const Uri &destination,
105 unsigned nstreams, DavixError **error)
106 {
107 d_ptr->copy(source, destination, nstreams, error);
108 }
109
110
111
112 void DavixCopy::setPerformanceCallback(PerformanceCallback callback, void *udata)
113 {
114 d_ptr->setPerformanceCallback(callback, udata);
115 }
116
117 void DavixCopy::setCancellationCallback(CancellationCallback callback, void *udata)
118 {
119 d_ptr->setCancellationCallback(callback, udata);
120 }
121
122
123 std::string DavixCopy::getTransferSourceHost() const
124 {
125 return d_ptr->getTransferSourceHost();
126 }
127
128 std::string DavixCopy::getTransferDestinationHost() const
129 {
130 return d_ptr->getTransferDestinationHost();
131 }
132
133
134 std::string DavixCopyInternal::getTransferSourceHost() const
135 {
136 return sourceHost;
137 }
138
139 std::string DavixCopyInternal::getTransferDestinationHost() const
140 {
141 return destinationHost;
142 }
143
144 void DavixCopyInternal::setTransferHost(const std::string& transferHost, bool activeParty) {
145 if (activeParty) {
146 if (parameters->getCopyMode() == CopyMode::Push) {
147 if (sourceHost.empty()) {
148 sourceHost = transferHost;
149 }
150 } else if (parameters->getCopyMode() == CopyMode::Pull) {
151 if (destinationHost.empty()) {
152 destinationHost = transferHost;
153 }
154 }
155 } else {
156 if (parameters->getCopyMode() == CopyMode::Push) {
157 if (destinationHost.empty()) {
158 destinationHost = transferHost;
159 }
160 } else if (parameters->getCopyMode() == CopyMode::Pull) {
161 if (sourceHost.empty()) {
162 sourceHost = transferHost;
163 }
164 }
165 }
166 }
167
168
169 void DavixCopyInternal::setPerformanceCallback(DavixCopy::PerformanceCallback callback,
170 void *udata)
171 {
172 perfCallback = callback;
173 perfCallbackUdata = udata;
174 }
175
176 void DavixCopyInternal::setCancellationCallback(DavixCopy::CancellationCallback callback, void *udata)
177 {
178 cancCallback = callback;
179 cancCallbackUdata = udata;
180 }
181
182 Uri dropDav(const Uri &uri) {
183 Uri retval(uri);
184
185 if(retval.getProtocol() == "dav") {
186 retval.setProtocol("http");
187 }
188 else if(retval.getProtocol() == "davs") {
189 retval.setProtocol("https");
190 }
191
192 return retval;
193 }
194
195 bool DavixCopyInternal::shouldCancel() {
196 if(!cancCallback) {
197 return false;
198 }
199
200 return cancCallback(cancCallbackUdata);
201 }
202
203 bool DavixCopyInternal::shouldCancel(Davix::DavixError **error) {
204 if(shouldCancel()) {
205 DavixError::clearError(error);
206 DavixError::setupError(error, COPY_SCOPE, StatusCode::Canceled, fmt::format("Request cancellation was requested."));
207 return true;
208 }
209
210 return false;
211 }
212
213 void DavixCopyInternal::copy(const Uri &src, const Uri &dst,
214 unsigned nstreams, DavixError **error)
215 {
216 std::string nextSrc, prevSrc, destination;
217 std::string delegationEndpoint;
218 DavixError *internalError = NULL;
219 bool suppressFinalHead = false;
220
221 Uri srcHttp = dropDav(src);
222 Uri dstHttp = dropDav(dst);
223
224 // set source and destination according to copy method
225 if(parameters->getCopyMode() == CopyMode::Push){
226 nextSrc = srcHttp.getString();
227 prevSrc = srcHttp.getString();
228 destination = dstHttp.getString();
229 }else if(parameters->getCopyMode() == CopyMode::Pull){
230 nextSrc = dstHttp.getString();
231 prevSrc = dstHttp.getString();
232 destination = srcHttp.getString();
233 }
234
235 // nstreams as string
236 char nstreamsStr[16];
237 snprintf(nstreamsStr, sizeof(nstreamsStr), "%u", nstreams);
238
239 // Need a copy so we can modify it
240 Davix::RequestParams requestParams(parameters);
241 requestParams.setTransparentRedirectionSupport(false);
242
243 // if destination is s3 endpoint, change prefix to http(s) and pre-sign the request as a PUT
244 if(destination.compare(0,2,"s3") == 0){
245 destination.replace(0, 2, "http");
246 Davix::HeaderVec vec;
247 Uri tmp;
248 std::string method = (parameters->getCopyMode() == CopyMode::Pull) ? "GET" : "PUT";
249 tmp = Davix::S3::signURI(requestParams, method, destination, vec, 3600);
250 destination = tmp.getString();
251 suppressFinalHead = true;
252 }
253
254 // handle gcloud endpoint as a destination
255 if(destination.compare(0,6,"gcloud") == 0){
256 destination.replace(0, 6, "http");
257
258 Davix::HeaderVec vec;
259 Uri tmp;
260 if (parameters->getCopyMode() == CopyMode::Pull)
261 tmp = Davix::gcloud::signURI(requestParams.getGcloudCredentials(), "GET", destination, vec, 3600);
262 else
263 tmp = Davix::gcloud::signURI(requestParams.getGcloudCredentials(), "PUT", destination, vec, 3600);
264 destination = tmp.getString();
265 suppressFinalHead = true;
266 }
267
268 if(destination.compare(0, 3, "dav") == 0) {
269 destination.replace(0, 3, "http");
270 }
271
272 if(destination.compare(0, 3, "cs3") == 0){
273 destination.replace(0, 3, "http");
274 }
275
276 // Perform COPY hopping through redirections
277 HttpRequest* request = NULL;
278 do {
279 nextSrc = _full_url(prevSrc, nextSrc);
280 prevSrc = nextSrc;
281 if (request) {
282 request->discardBody(&internalError);
283 if (!internalError)
284 request->endRequest(&internalError);
285 if (internalError) {
286 DavixError::propagatePrefixedError(error, internalError, __func__);
287 break;
288 }
289 }
290 delete request;
291
292 DAVIX_SLOG(DAVIX_LOG_VERBOSE, DAVIX_LOG_GRID, "Hop: {}",
293 nextSrc);
294
295 request = context.createRequest(nextSrc, &internalError);
296 if (internalError) {
297 DavixError::propagatePrefixedError(error, internalError, __func__);
298 break;
299 }
300
301 request->setRequestMethod("COPY");
302 if(parameters->getCopyMode() == CopyMode::Push){
303 request->addHeaderField("Destination", destination);
304 }else if(parameters->getCopyMode() == CopyMode::Pull){
305 request->addHeaderField("Source", destination);
306 }
307 request->addHeaderField("X-Number-Of-Streams", nstreamsStr);
308
309 // for lcgdm-dav, ask for secure redirection in all cases for COPY
310 request->addHeaderField("Secure-Redirection", "1");
311
312 // for lcgdm-dav -> S3, add NoHead flag to suppress final head-to-close request
313 if(suppressFinalHead)
314 request->addHeaderField("Copy-Flags", "NoHead");
315 request->setParameters(requestParams);
316 request->beginRequest(&internalError);
317 if (internalError) {
318 DavixError::propagatePrefixedError(error, internalError, __func__);
319 break;
320 }
321
322 // If we get a X-Delegate-To, before continuing, delegate
323 if (request->getAnswerHeader("X-Delegate-To", delegationEndpoint)) {
324 std::vector<std::string> delegationEndpoints = parseXDelegateTo(nextSrc, delegationEndpoint, &internalError);
325 if (internalError) {
326 DavixError::propagatePrefixedError(error, internalError, __func__);
327 break;
328 }
329
330 DAVIX_SLOG(DAVIX_LOG_VERBOSE, DAVIX_LOG_GRID, "Got delegation endpoint: {}",
331 delegationEndpoint.c_str());
332
333 std::string dlg_id = DavixDelegation::delegate(context, delegationEndpoints,
334 parameters, &internalError);
335 if (internalError) {
336 DavixError::propagatePrefixedError(error, internalError, __func__);
337 break;
338 }
339
340 DAVIX_SLOG(DAVIX_LOG_VERBOSE, DAVIX_LOG_GRID, "Got delegation ID {}",
341 dlg_id.c_str());
342
343 dlg_id.clear();
344 }
345
346 } while (!shouldCancel() && request->getAnswerHeader("Location", nextSrc) && request->getRequestCode() >= 300 && request->getRequestCode() < 400);
347
348 if (!*error) {
349 int responseStatus = request->getRequestCode();
350 if (responseStatus == 404) {
351 DavixError::setupError(error, COPY_SCOPE, StatusCode::FileNotFound,
352 "Could not COPY. File not found");
353 }
354 else if (responseStatus == 403) {
355 DavixError::setupError(error, COPY_SCOPE, StatusCode::PermissionRefused,
356 "Could not COPY. Permission denied.");
357 }
358 else if (responseStatus == 501) {
359 DavixError::setupError(error, COPY_SCOPE, StatusCode::OperationNonSupported,
360 "Could not COPY. The source service does not support it");
361 }
362 else if (responseStatus >= 405) {
363 DavixError::setupError(error, COPY_SCOPE, StatusCode::OperationNonSupported,
364 "Could not COPY. The source service does not allow it");
365 }
366 else if (responseStatus == 400) {
367 std::string err_msg(request->getAnswerContentVec().begin(), request->getAnswerContentVec().end());
368 DavixError::setupError(error, COPY_SCOPE, StatusCode::InvalidArgument,
369 fmt::format("Could not COPY. The server rejected the request: {}", err_msg));
370 }
371 else if (responseStatus >= 300) {
372 std::ostringstream msg;
373 msg << "Could not COPY. Unknown error code: " << responseStatus;
374 DavixError::setupError(error, COPY_SCOPE, StatusCode::UnknownError,
375 msg.str());
376 }
377 }
378
379 if(shouldCancel(error)) {
380 return;
381 }
382
383 // Did we fail?
384 if (*error)
385 return;
386
387 // Finished hopping
388 setTransferHost(Uri(nextSrc).getHost(), true);
389
390 // Just wait for it to finish
391 monitorPerformanceMarkers(request, error);
392 request->endRequest(&internalError);
393
394 if(internalError && !(*error) ) {
395 DavixError::propagatePrefixedError(error, internalError, __func__);
396 }
397
398 if(shouldCancel(error)) {
399 return;
400 }
401
402 delete request;
403 }
404
405 // Parse "RemoteConnections" line and extract IP string
406 // Example: tcp:[fff:aaa:192:168::100:e]:443 --> [fff:aaa:192:168::100:e]
407 // Example: udp:192.168.1.100:8443 --> 192.168.1.100
408 std::string getIPString(char *text) {
409 int idx = 0;
410
411 // Skip any initial whitespaces
412 while (text[idx] && isspace(text[idx])) {
413 idx++;
414 }
415
416 if (!text[idx]) {
417 return "";
418 }
419
420 if ((strncasecmp("tcp:", text + idx, 4) != 0) &&
421 (strncasecmp("udp:", text + idx, 4) != 0)) {
422 return "";
423 }
424
425 if (!text[idx + 4]) {
426 return "";
427 }
428
429 std::string ipstring = text + idx + 4;
430 size_t endpos;
431
432 if (ipstring.back() == '\n') {
433 ipstring.pop_back();
434 }
435
436 // Trim string to only the IP notation
437 if (ipstring.front() == '[') {
438 // Dealing with IPv6 scenario
439 endpos = ipstring.find(']');
440
441 if (endpos == std::string::npos) {
442 return "";
443 }
444
445 endpos += 1;
446 } else {
447 endpos = ipstring.find(':');
448
449 if (endpos == std::string::npos) {
450 endpos = ipstring.size();
451 }
452 }
453
454 ipstring = ipstring.substr(0, endpos);
455 return ipstring;
456 }
457
458 // Parse string to determine IP version
459 enum IPtype getIPType(char *text) {
460 int lbkt = 0, rbkt = 0, per = 0;
461 std::string ipstring = getIPString(text);
462
463 for (const char& c: ipstring) {
464 if (c == '[') {
465 lbkt++;
466 } else if (c == ']') {
467 rbkt++;
468 } else if (c == '.') {
469 per++;
470 }
471 }
472
473 /*
474 * Test for IPv4 or IPv6 address. IPv6 type contains square brackets. IPv4 contains 3 periods.
475 * Note that IPv4-mapped IPv6 addresses are of the format [::ffff.<IPv4 address>]
476 */
477 if (lbkt && rbkt) {
478 return IPtype::IPv6;
479 } else if (per == 3) {
480 return IPtype::IPv4;
481 }
482
483 return IPtype::undefined;
484 }
485
486 void logPerfmarker(const std::list<std::string>& lines)
487 {
488 std::ostringstream out;
489 std::copy(lines.begin(), lines.end(),std::ostream_iterator<std::string>(out, "\n"));
490 DAVIX_SLOG(DAVIX_LOG_VERBOSE, DAVIX_LOG_GRID, "PerformanceMarker:\n{}", out.str());
491 }
492
493 void DavixCopyInternal::monitorPerformanceMarkers(Davix::HttpRequest *request,
494 Davix::DavixError **error)
495 {
496 Davix::DavixError* daverr = NULL;
497 char buffer[1024], *p;
498 dav_ssize_t line_len;
499
500 PerformanceMarker holder;
501 PerformanceData performance;
502 std::list<std::string> perfmarkerLines;
503 time_t lastPerfCallback = 0;
504 bool clearOutcome = false;
505
506 while ((line_len = request->readLine(buffer, sizeof(buffer), &daverr)) > 0 && !daverr && !shouldCancel())
507 {
508 buffer[line_len] = '\0';
509
510 // Skip trailing whitespaces and newlines
511 p = buffer + line_len - 1;
512 while (p > buffer && isspace(*p))
513 --p;
514 *(p + 1) = '\0';
515
516 if (line_len > 0) {
517 perfmarkerLines.emplace_back(buffer);
518 }
519
520 // Skip heading whitespaces
521 p = buffer;
522 while (*p && p < buffer + sizeof(buffer) && isspace(*p))
523 ++p;
524
525 if (strncasecmp("Perf Marker", p, 11) == 0)
526 {
527 holder = PerformanceMarker();
528 }
529 else if (strncasecmp("Timestamp:", p, 10) == 0)
530 {
531 holder.latest = atol(p + 10);
532 }
533 else if (strncasecmp("Stripe Index:", p, 13) == 0)
534 {
535 holder.index = atoi(p + 13);
536 }
537 else if (strncasecmp("Stripe Bytes Transferred:", p, 25) == 0)
538 {
539 holder.transferred = atol(p + 26);
540 }
541 else if (strncasecmp("Total Stripe Count:", p, 19) == 0)
542 {
543 holder.count = atoi(p + 20);
544 }
545 else if (strncasecmp("RemoteConnections:", p, 18) == 0)
546 {
547 std::string ipstring = getIPString(p + 18);
548 setTransferHost(ipstring, false);
549 performance.ipflag = getIPType(p + 18);
550 DAVIX_SLOG(DAVIX_LOG_DEBUG, DAVIX_LOG_GRID, "Got IP: {} (ipver={})", ipstring, performance.ipflag);
551 }
552 else if (strncasecmp("End", p, 3) == 0)
553 {
554 performance.update(holder);
555 time_t now = time(NULL);
556 if (now - lastPerfCallback >= 1)
557 {
558 if (this->perfCallback)
559 this->perfCallback(performance, this->perfCallbackUdata);
560 lastPerfCallback = now;
561 }
562 logPerfmarker(perfmarkerLines);
563 perfmarkerLines.clear();
564 }
565 else if (strncasecmp("success", p, 7) == 0)
566 {
567 clearOutcome = true;
568 request->discardBody(&daverr);
569 logPerfmarker(perfmarkerLines);
570 break;
571 }
572 else if (strncasecmp("aborted", p, 7) == 0)
573 {
574 clearOutcome = true;
575 Davix::DavixError::setupError(error, COPY_SCOPE, StatusCode::Canceled,
576 "Transfer aborted in the remote end");
577 logPerfmarker(perfmarkerLines);
578 break;
579 }
580 else if (strncasecmp("failed", p, 6) == 0 || strncasecmp("failure", p, 7) == 0)
581 {
582 clearOutcome = true;
583 Davix::DavixError::setupError(error, COPY_SCOPE, StatusCode::RemoteError,
584 std::string("Transfer ") + p);
585 logPerfmarker(perfmarkerLines);
586 break;
587 }
588 else if (line_len > 0)
589 {
590 DAVIX_SLOG(DAVIX_LOG_DEBUG, DAVIX_LOG_GRID, "Unknown performance marker, ignoring: {}", buffer);
591 }
592 }
593
594 if(!clearOutcome && !(*error)) {
595 Davix::DavixError::setupError(error, COPY_SCOPE, StatusCode::UnknownError,
596 std::string("Connection terminated abruptly; Status of TPC request unknown"));
597 }
598 }