| 1 |
/* |
| 2 |
* This File is part of Davix, The IO library for HTTP based protocols |
| 3 |
* Copyright (C) CERN 2013 |
| 4 |
* Author: Adrien Devresse <[email protected]> |
| 5 |
* |
| 6 |
* This library is free software; you can redistribute it and/or |
| 7 |
* modify it under the terms of the GNU Lesser General Public |
| 8 |
* License as published by the Free Software Foundation; either |
| 9 |
* version 2.1 of the License, or (at your option) any later version. |
| 10 |
* |
| 11 |
* This library is distributed in the hope that it will be useful, |
| 12 |
* but WITHOUT ANY WARRANTY; without even the implied warranty of |
| 13 |
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU |
| 14 |
* Lesser General Public License for more details. |
| 15 |
* |
| 16 |
* You should have received a copy of the GNU Lesser General Public |
| 17 |
* License along with this library; if not, write to the Free Software |
| 18 |
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA |
| 19 |
* |
| 20 |
*/ |
| 21 |
|
| 22 |
#include <davix_internal.hpp> |
| 23 |
#include "davmeta.hpp" |
| 24 |
|
| 25 |
#include <xml/davpropxmlparser.hpp> |
| 26 |
#include <xml/davdeletexmlparser.hpp> |
| 27 |
#include <xml/metalinkparser.hpp> |
| 28 |
#include <xml/s3propparser.hpp> |
| 29 |
#include <xml/azurepropparser.hpp> |
| 30 |
#include <xml/swiftpropparser.hpp> |
| 31 |
|
| 32 |
#include <utils/davix_logger_internal.hpp> |
| 33 |
#include <utils/davix_utils_internal.hpp> |
| 34 |
#include <utils/davix_s3_utils.hpp> |
| 35 |
#include <utils/davix_azure_utils.hpp> |
| 36 |
#include <utils/davix_gcloud_utils.hpp> |
| 37 |
#include <utils/davix_swift_utils.hpp> |
| 38 |
#include <utils/checksum_extractor.hpp> |
| 39 |
|
| 40 |
#include <request/httprequest.hpp> |
| 41 |
#include <fileops/fileutils.hpp> |
| 42 |
#include <utils/stringutils.hpp> |
| 43 |
#include "libs/alibxx/crypto/base64.hpp" |
| 44 |
#include <neon/neonrequest.hpp> |
| 45 |
|
| 46 |
|
| 47 |
using namespace StrUtil; |
| 48 |
|
| 49 |
namespace Davix{ |
| 50 |
|
| 51 |
static const std::string stat_listing("<?xml version=\"1.0\" encoding=\"utf-8\" ?><D:propfind xmlns:D=\"DAV:\" xmlns:L=\"LCGDM:\"><D:prop>" |
| 52 |
"<D:displayname/><D:getlastmodified/><D:creationdate/><D:getcontentlength/><D:quota-used-bytes/>" |
| 53 |
"<D:resourcetype><D:collection/></D:resourcetype><L:mode/>" |
| 54 |
"<D:owner></D:owner><D:group></D:group>" |
| 55 |
"</D:prop>" |
| 56 |
"</D:propfind>"); |
| 57 |
|
| 58 |
static const std::string quota_stat("<?xml version=\"1.0\" encoding=\"utf-8\" ?><D:propfind xmlns:D=\"DAV:\" xmlns:L=\"LCGDM:\"><D:prop>" |
| 59 |
"<D:quota-used-bytes/><D:quota-available-bytes/>" |
| 60 |
"</D:prop>" |
| 61 |
"</D:propfind>"); |
| 62 |
|
| 63 |
struct DirHandle{ |
| 64 |
|
| 65 |
DirHandle(HttpRequest* req, XMLPropParser * p): request(req), parser(p){} |
| 66 |
|
| 67 |
std::unique_ptr<HttpRequest> request; |
| 68 |
std::unique_ptr<Davix::XMLPropParser> parser; |
| 69 |
|
| 70 |
|
| 71 |
}; |
| 72 |
|
| 73 |
/** |
| 74 |
* Helper function to execute a PROPFIND/stat request on a given HTTP request handle. |
| 75 |
* Returns a string with the HTTP response. |
| 76 |
*/ |
| 77 |
std::string req_webdav_propfind(HttpRequest* req, DavixError** err) { |
| 78 |
req->addHeaderField("Depth", "0"); |
| 79 |
req->setRequestMethod("PROPFIND"); |
| 80 |
|
| 81 |
if (req->executeRequest(err) == 0) { |
| 82 |
auto content = req->getAnswerContent(); |
| 83 |
|
| 84 |
if (content != NULL) { |
| 85 |
return {content}; |
| 86 |
} |
| 87 |
} |
| 88 |
|
| 89 |
return {}; |
| 90 |
} |
| 91 |
|
| 92 |
|
| 93 |
int dav_stat_mapper_webdav(Context &context, const RequestParams* params, const Uri & url, struct StatInfo& st_info){ |
| 94 |
DavPropXMLParser parser; |
| 95 |
DavixError* tmp_err = NULL; |
| 96 |
HttpRequest req(context, url, &tmp_err); |
| 97 |
int ret = -1; |
| 98 |
|
| 99 |
if( tmp_err == NULL){ |
| 100 |
req.setParameters(params); |
| 101 |
|
| 102 |
TRY_DAVIX { |
| 103 |
std::string response = req_webdav_propfind(&req, &tmp_err); |
| 104 |
|
| 105 |
if (!tmp_err) { |
| 106 |
parser.parseChunk(response); |
| 107 |
std::deque<FileProperties>& props = parser.getProperties(); |
| 108 |
|
| 109 |
if (props.size() < 1) { |
| 110 |
throw DavixException(davix_scope_stat_str(), Davix::StatusCode::WebDavPropertiesParsingError, "Parsing Error: properties number < 1"); |
| 111 |
} else { |
| 112 |
st_info = props.front().info; |
| 113 |
ret = 0; |
| 114 |
} |
| 115 |
} |
| 116 |
} CATCH_DAVIX(&tmp_err) |
| 117 |
|
| 118 |
if (tmp_err != NULL) { |
| 119 |
ret = -1; |
| 120 |
} |
| 121 |
} |
| 122 |
|
| 123 |
checkDavixError(&tmp_err); |
| 124 |
return ret; |
| 125 |
} |
| 126 |
|
| 127 |
|
| 128 |
int dav_stat_mapper_http(Context& context, const RequestParams* params, const Uri & uri, struct StatInfo& st_info){ |
| 129 |
int ret = -1; |
| 130 |
DavixError * tmp_err=NULL; |
| 131 |
HeadRequest req(context, uri, &tmp_err); |
| 132 |
|
| 133 |
if( tmp_err == NULL){ |
| 134 |
req.setParameters(params); |
| 135 |
req.executeRequest(&tmp_err); |
| 136 |
|
| 137 |
if(!tmp_err){ |
| 138 |
if(httpcodeIsValid(req.getRequestCode()) ){ |
| 139 |
memset(&st_info, 0, sizeof(struct StatInfo)); |
| 140 |
const dav_ssize_t s = req.getAnswerSize(); |
| 141 |
st_info.size = std::max<dav_ssize_t>(0,s); |
| 142 |
st_info.mode = 0755 | S_IFREG; |
| 143 |
ret = 0; |
| 144 |
}else{ |
| 145 |
httpcodeToDavixError(req.getRequestCode(), davix_scope_http_request(), uri.getString() , &tmp_err); |
| 146 |
ret = -1; |
| 147 |
} |
| 148 |
} |
| 149 |
} |
| 150 |
checkDavixError(&tmp_err); |
| 151 |
return ret; |
| 152 |
} |
| 153 |
|
| 154 |
|
| 155 |
// Implement stat with a GET of Range 1 |
| 156 |
int dav_stat_mapper_http_get(Context& context, const RequestParams* params, const Uri & uri, struct StatInfo& st_info){ |
| 157 |
int ret = -1; |
| 158 |
DavixError * tmp_err=NULL; |
| 159 |
GetRequest req(context, uri, &tmp_err); |
| 160 |
|
| 161 |
if( tmp_err == NULL){ |
| 162 |
req.setParameters(params); |
| 163 |
req.addHeaderField("Range", "bytes=0-1"); |
| 164 |
req.executeRequest(&tmp_err); |
| 165 |
|
| 166 |
if(!tmp_err){ |
| 167 |
if(httpcodeIsValid(req.getRequestCode()) ){ |
| 168 |
memset(&st_info, 0, sizeof(struct StatInfo)); |
| 169 |
std::string rnge; |
| 170 |
req.getAnswerHeader("Content-Range", rnge); |
| 171 |
std::string::size_type pos = rnge.find_first_of("/"); |
| 172 |
if (pos == std::string::npos) { |
| 173 |
throw DavixException(davix_scope_meta(), StatusCode::ParsingError, "Content-Range not parsable"); |
| 174 |
} |
| 175 |
if (rnge.substr(pos+1,1) == "*") { |
| 176 |
throw DavixException(davix_scope_meta(), StatusCode::ParsingError, "Server does not provide content length"); |
| 177 |
} |
| 178 |
long lsize = toType<long, std::string>()(rnge.substr(pos+1)); |
| 179 |
st_info.size = std::max<long>(0,lsize); |
| 180 |
st_info.mode = 0755 | S_IFREG; |
| 181 |
req.discardBody(&tmp_err); |
| 182 |
ret = 0; |
| 183 |
}else{ |
| 184 |
httpcodeToDavixError(req.getRequestCode(), davix_scope_http_request(), uri.getString() , &tmp_err); |
| 185 |
ret = -1; |
| 186 |
} |
| 187 |
} |
| 188 |
} |
| 189 |
checkDavixError(&tmp_err); |
| 190 |
return ret; |
| 191 |
} |
| 192 |
|
| 193 |
|
| 194 |
dav_ssize_t incremental_listdir_parsing(HttpRequest* req, XMLPropParser * parser, dav_size_t s_buff, const std::string & scope){ |
| 195 |
DavixError* tmp_err=NULL; |
| 196 |
|
| 197 |
char buffer[s_buff+1]; |
| 198 |
const dav_ssize_t ret = req->readSegment(buffer, s_buff, &tmp_err); |
| 199 |
checkDavixError(&tmp_err); |
| 200 |
if(ret >= 0){ |
| 201 |
buffer[ret]= '\0'; |
| 202 |
parser->parseChunk(buffer, ret); |
| 203 |
}else{ |
| 204 |
throw DavixException(scope, StatusCode::UnknownError, "Unknown readSegment error"); |
| 205 |
} |
| 206 |
|
| 207 |
return ret; |
| 208 |
} |
| 209 |
|
| 210 |
|
| 211 |
dav_ssize_t getStatInfo(Context & c, const Uri & url, const RequestParams * p, |
| 212 |
struct StatInfo& st_info){ |
| 213 |
RequestParams params(p); |
| 214 |
configureRequestParamsProto(url, params); |
| 215 |
int ret =-1; |
| 216 |
|
| 217 |
switch(params.getProtocol()){ |
| 218 |
case RequestProtocol::Webdav: |
| 219 |
ret = dav_stat_mapper_webdav(c, ¶ms, url, st_info); |
| 220 |
break; |
| 221 |
default: |
| 222 |
if (isS3SignedURL(url)) { |
| 223 |
// This endpoint won't accept a HEAD request, use GET instead |
| 224 |
ret = dav_stat_mapper_http_get(c, ¶ms, url, st_info); |
| 225 |
} else { |
| 226 |
ret = dav_stat_mapper_http(c, ¶ms, url, st_info); |
| 227 |
} |
| 228 |
break; |
| 229 |
|
| 230 |
} |
| 231 |
DAVIX_SLOG(DAVIX_LOG_DEBUG, DAVIX_LOG_CHAIN, " davix_stat <-"); |
| 232 |
return ret; |
| 233 |
} |
| 234 |
|
| 235 |
QuotaInfo::QuotaInfo() : d_ptr(new Internal()) { } |
| 236 |
QuotaInfo::~QuotaInfo() { } |
| 237 |
dav_size_t QuotaInfo::getUsedBytes() { |
| 238 |
return d_ptr->used_bytes; |
| 239 |
} |
| 240 |
dav_size_t QuotaInfo::getFreeSpace() { |
| 241 |
return d_ptr->free_space; |
| 242 |
} |
| 243 |
|
| 244 |
class QuotaInfoHandler { |
| 245 |
public: |
| 246 |
static void setdptr(QuotaInfo &info, QuotaInfo::Internal &internal) { |
| 247 |
info.d_ptr.reset(new QuotaInfo::Internal(internal)); |
| 248 |
} |
| 249 |
}; |
| 250 |
|
| 251 |
void getQuotaInfo(Context & c, const Uri & url, const RequestParams *p, QuotaInfo &info) { |
| 252 |
DavixError * tmp_err = NULL; |
| 253 |
HttpRequest req(c, url, &tmp_err); |
| 254 |
checkDavixError(&tmp_err); |
| 255 |
|
| 256 |
req.setParameters(*p); |
| 257 |
req.addHeaderField("Depth","0"); |
| 258 |
req.setRequestMethod("PROPFIND"); |
| 259 |
req.setRequestBody(quota_stat); |
| 260 |
|
| 261 |
if (req.executeRequest(&tmp_err) == 0 && !tmp_err) { |
| 262 |
DavPropXMLParser parser; |
| 263 |
auto content = req.getAnswerContent(); |
| 264 |
std::string response = (content != NULL) ? content : ""; |
| 265 |
parser.parseChunk(response); |
| 266 |
std::deque<FileProperties> & props = parser.getProperties(); |
| 267 |
|
| 268 |
if (props.size() < 1) { |
| 269 |
throw DavixException(davix_scope_stat_str(), Davix::StatusCode::WebDavPropertiesParsingError, "Parsing Error: properties number < 1"); |
| 270 |
} else { |
| 271 |
QuotaInfoHandler::setdptr(info, props.front().quota); |
| 272 |
} |
| 273 |
} |
| 274 |
|
| 275 |
checkDavixError(&tmp_err); |
| 276 |
} |
| 277 |
|
| 278 |
void parse_creation_deletion_result(int code, const Uri & u, const std::string & scope, const std::vector<char> & body){ |
| 279 |
switch(code) { |
| 280 |
case 200: |
| 281 |
case 201: |
| 282 |
case 202: |
| 283 |
case 204: { |
| 284 |
return; |
| 285 |
} |
| 286 |
case 207: { |
| 287 |
// Parse WebDAV |
| 288 |
DavDeleteXMLParser parser; |
| 289 |
std::string response = (!body.empty()) ? body.data() : ""; |
| 290 |
parser.parseChunk(response); |
| 291 |
|
| 292 |
if (parser.getProperties().size() > 0) { |
| 293 |
for (unsigned int i = 0; i < parser.getProperties().size(); ++i) { |
| 294 |
const int sub_code = parser.getProperties().at(i).req_status; |
| 295 |
std::ostringstream ss; |
| 296 |
ss << "occurred during deletion request for " << parser.getProperties().at(i).filename; |
| 297 |
|
| 298 |
if (httpcodeIsValid(sub_code) == false) { |
| 299 |
httpcodeToDavixException(sub_code, scope, ss.str()); |
| 300 |
} |
| 301 |
} |
| 302 |
|
| 303 |
return; |
| 304 |
} |
| 305 |
|
| 306 |
// if no properties, properties were filtered because invalid |
| 307 |
httpcodeToDavixException(404, scope); |
| 308 |
break; |
| 309 |
} |
| 310 |
} |
| 311 |
|
| 312 |
std::ostringstream ss; |
| 313 |
ss << " with url " << u.getString(); |
| 314 |
httpcodeToDavixException(code, scope, ss.str()); |
| 315 |
} |
| 316 |
|
| 317 |
|
| 318 |
int internal_delete_resource(Context & c, const Uri & url, const RequestParams & params){ |
| 319 |
DavixError* tmp_err=NULL; |
| 320 |
int ret=-1; |
| 321 |
RequestParams _params(params); |
| 322 |
|
| 323 |
DeleteRequest req(c, url, &tmp_err); |
| 324 |
req.setParameters(_params); |
| 325 |
if(!tmp_err){ |
| 326 |
if( ( ret=req.executeRequest(&tmp_err)) == 0){ |
| 327 |
parse_creation_deletion_result(req.getRequestCode(), url, davix_scope_rm_str(), req.getAnswerContentVec()); |
| 328 |
} |
| 329 |
} |
| 330 |
|
| 331 |
|
| 332 |
checkDavixError(&tmp_err); |
| 333 |
return ret; |
| 334 |
} |
| 335 |
|
| 336 |
|
| 337 |
int internal_make_collection(Context & c, const Uri & url, const RequestParams & params){ |
| 338 |
DAVIX_SCOPE_TRACE(DAVIX_LOG_CHAIN, mkcoll); |
| 339 |
|
| 340 |
int ret=-1; |
| 341 |
DavixError* tmp_err=NULL; |
| 342 |
RequestParams _params(params); |
| 343 |
|
| 344 |
HttpRequest req(c, url, &tmp_err); |
| 345 |
|
| 346 |
if(tmp_err == NULL){ |
| 347 |
req.setParameters(params); |
| 348 |
req.setRequestMethod("MKCOL"); |
| 349 |
if( (ret = req.executeRequest(&tmp_err)) == 0){ |
| 350 |
parse_creation_deletion_result(req.getRequestCode(), url, davix_scope_mkdir_str(), req.getAnswerContentVec()); |
| 351 |
} |
| 352 |
|
| 353 |
} |
| 354 |
|
| 355 |
checkDavixError(&tmp_err); |
| 356 |
return ret; |
| 357 |
} |
| 358 |
|
| 359 |
|
| 360 |
int internal_move(Context & c, const Uri & url, const RequestParams & params, const std::string & target_url){ |
| 361 |
DAVIX_SLOG(DAVIX_LOG_DEBUG, DAVIX_LOG_CHAIN, " -> move"); |
| 362 |
int ret=-1; |
| 363 |
DavixError* tmp_err=NULL; |
| 364 |
RequestParams _params(params); |
| 365 |
|
| 366 |
HttpRequest req(c, url, &tmp_err); |
| 367 |
|
| 368 |
if(tmp_err == NULL){ |
| 369 |
req.setParameters(params); |
| 370 |
req.setRequestMethod("MOVE"); |
| 371 |
|
| 372 |
Uri uri(target_url); |
| 373 |
uri.httpizeProtocol(); |
| 374 |
req.addHeaderField("Destination", uri.getString()); |
| 375 |
|
| 376 |
if( (ret = req.executeRequest(&tmp_err)) == 0){ |
| 377 |
parse_creation_deletion_result(req.getRequestCode(), url, davix_scope_mv_str(), req.getAnswerContentVec()); |
| 378 |
} |
| 379 |
|
| 380 |
} |
| 381 |
|
| 382 |
DAVIX_SLOG(DAVIX_LOG_DEBUG, DAVIX_LOG_CHAIN, " move <-"); |
| 383 |
checkDavixError(&tmp_err); |
| 384 |
return ret; |
| 385 |
} |
| 386 |
|
| 387 |
|
| 388 |
|
| 389 |
int internal_checksum(Context & c, const Uri & url, const RequestParams *p, std::string & checksm, const std::string & chk_algo){ |
| 390 |
DAVIX_SLOG(DAVIX_LOG_DEBUG, DAVIX_LOG_CHAIN, " -> checksum"); |
| 391 |
int ret=-1; |
| 392 |
DavixError* tmp_err=NULL; |
| 393 |
RequestParams params(p); |
| 394 |
|
| 395 |
HeadRequest req(c, url, &tmp_err); |
| 396 |
|
| 397 |
if(tmp_err == NULL){ |
| 398 |
// add Digest file, support for other digest, extended format |
| 399 |
req.addHeaderField("Want-Digest", chk_algo); |
| 400 |
req.setParameters(params); |
| 401 |
if( (ret = req.executeRequest(&tmp_err)) == 0 |
| 402 |
&& !tmp_err && (ret = davixRequestToFileStatus(&req, davix_scope_mkdir_str(), &tmp_err)) >=0){ |
| 403 |
|
| 404 |
// try simple MD5 ( standard ) |
| 405 |
if(compare_ncase(chk_algo, "MD5") == 0){ |
| 406 |
std::string chk; |
| 407 |
if(req.getAnswerHeader("Content-MD5", chk) == true){ |
| 408 |
DAVIX_SLOG(DAVIX_LOG_TRACE, DAVIX_LOG_CHAIN, "Extract MD5 checksum in base64 {}", chk); |
| 409 |
chk= Base64::base64_decode(chk); |
| 410 |
std::swap(checksm, chk); |
| 411 |
return 0; |
| 412 |
} |
| 413 |
} |
| 414 |
|
| 415 |
// fallback on extension for checksum |
| 416 |
HeaderVec headers; |
| 417 |
req.getAnswerHeaders(headers); |
| 418 |
|
| 419 |
if(ChecksumExtractor::extractChecksum(headers, chk_algo, checksm)) { |
| 420 |
return 0; |
| 421 |
} |
| 422 |
|
| 423 |
// last chance try to extract MD5 checksum from ETAG ( S3 work around ) |
| 424 |
std::string etag_str; |
| 425 |
if(compare_ncase(chk_algo, "MD5") ==0 && req.getAnswerHeader("etag", etag_str)){ |
| 426 |
stringVec tokens = tokenSplit(etag_str, "&;\\/\"'"); |
| 427 |
for(stringVec::iterator it = tokens.begin(); it < tokens.end(); it++){ |
| 428 |
if(it->size() == 32 && std::find_if(it->begin(), it->end(), std::not1(StrUtil::isHexa())) == it->end()){ |
| 429 |
std::swap(checksm, *it); |
| 430 |
return 0; |
| 431 |
} |
| 432 |
} |
| 433 |
} |
| 434 |
|
| 435 |
std::ostringstream ss; |
| 436 |
ss << "checksum calculation for " << chk_algo << " not supported for " << url; |
| 437 |
throw DavixException(davix_scope_meta(), StatusCode::OperationNonSupported, ss.str()); |
| 438 |
|
| 439 |
DAVIX_SLOG(DAVIX_LOG_DEBUG, DAVIX_LOG_CHAIN, " checksum <-"); |
| 440 |
return 0; |
| 441 |
} |
| 442 |
} |
| 443 |
throw DavixException(&tmp_err); |
| 444 |
} |
| 445 |
|
| 446 |
|
| 447 |
|
| 448 |
|
| 449 |
bool wedav_get_next_property(std::unique_ptr<DirHandle> & handle, std::string & name_entry, StatInfo & info){ |
| 450 |
DAVIX_SLOG(DAVIX_LOG_DEBUG, DAVIX_LOG_CHAIN, " -> wedav_get_next_property"); |
| 451 |
const size_t read_size = 2048; |
| 452 |
|
| 453 |
|
| 454 |
HttpRequest& req = *(handle->request); // setup env again |
| 455 |
XMLPropParser& parser = *(handle->parser); |
| 456 |
|
| 457 |
size_t prop_size = parser.getProperties().size(); |
| 458 |
ssize_t s_resu = read_size; |
| 459 |
|
| 460 |
while( prop_size == 0 |
| 461 |
&& s_resu > 0){ // request not complete and current data too smalls |
| 462 |
// continue the parsing until one more result |
| 463 |
s_resu = incremental_listdir_parsing(&req, &parser, read_size, "WebDav::listing"); |
| 464 |
|
| 465 |
prop_size = parser.getProperties().size(); |
| 466 |
} |
| 467 |
|
| 468 |
|
| 469 |
if(prop_size == 0){ |
| 470 |
return false; // end of the request, end of the story |
| 471 |
} |
| 472 |
|
| 473 |
FileProperties & front = parser.getProperties().front(); |
| 474 |
name_entry.swap(front.filename); |
| 475 |
info = front.info; |
| 476 |
parser.getProperties().pop_front(); // clean the current element |
| 477 |
return true; |
| 478 |
} |
| 479 |
|
| 480 |
|
| 481 |
void webdav_start_listing_query(std::unique_ptr<DirHandle> & handle, Context & context, const RequestParams* params, const Uri & url, const std::string & body){ |
| 482 |
dav_ssize_t s_resu; |
| 483 |
|
| 484 |
DavixError* tmp_err=NULL; |
| 485 |
handle.reset(new DirHandle(new PropfindRequest(context, url, &tmp_err), new DavPropXMLParser())); |
| 486 |
checkDavixError(&tmp_err); |
| 487 |
|
| 488 |
HttpRequest & http_req = *(handle->request); |
| 489 |
XMLPropParser & parser = *(handle->parser); |
| 490 |
|
| 491 |
http_req.addHeaderField("Depth","1"); |
| 492 |
http_req.setParameters(params); |
| 493 |
// setup the handle for simple listing only |
| 494 |
http_req.setRequestBody(body); |
| 495 |
|
| 496 |
http_req.beginRequest(&tmp_err); |
| 497 |
checkDavixError(&tmp_err); |
| 498 |
|
| 499 |
check_file_status(http_req, davix_scope_directory_listing_str()); |
| 500 |
|
| 501 |
size_t prop_size = 0; |
| 502 |
do{ // parse the begining of the request until the first property -> directory property |
| 503 |
s_resu = incremental_listdir_parsing(&http_req, &parser, 2048, davix_scope_directory_listing_str()); |
| 504 |
|
| 505 |
prop_size = parser.getProperties().size(); |
| 506 |
if(s_resu < 2048 && prop_size <1){ // verify request status : if req done + no data -> error |
| 507 |
throw DavixException(davix_scope_directory_listing_str(), StatusCode::WebDavPropertiesParsingError, "bad server answer, not a valid WebDav PROPFIND answer"); |
| 508 |
} |
| 509 |
|
| 510 |
}while( prop_size < 1); // leave is end of req & no data |
| 511 |
|
| 512 |
const StatInfo & info = parser.getProperties().at(0).info; |
| 513 |
if( S_ISDIR(info.mode) == false){ |
| 514 |
std::ostringstream ss; |
| 515 |
ss << url << " is not a collection, listing impossible"; |
| 516 |
throw DavixException(davix_scope_directory_listing_str(), StatusCode::IsNotADirectory, ss.str()); |
| 517 |
}else{ |
| 518 |
parser.getProperties().pop_front(); // suppress the parent directory infos... |
| 519 |
} |
| 520 |
|
| 521 |
} |
| 522 |
|
| 523 |
bool webdav_directory_listing(std::unique_ptr<DirHandle> & handle, Context & context, const RequestParams* params, const Uri & uri, const std::string & body, std::string & name_entry, StatInfo & info){ |
| 524 |
if(handle.get() == NULL){ |
| 525 |
webdav_start_listing_query(handle, context, params, uri, body); |
| 526 |
} |
| 527 |
return wedav_get_next_property(handle, name_entry, info); |
| 528 |
} |
| 529 |
|
| 530 |
HttpMetaOps::HttpMetaOps(): HttpIOChain(){} |
| 531 |
|
| 532 |
HttpMetaOps::~HttpMetaOps(){} |
| 533 |
|
| 534 |
|
| 535 |
void HttpMetaOps::checksum(IOChainContext & iocontext, std::string &checksm, const std::string &chk_algo){ |
| 536 |
internal_checksum(iocontext._context, iocontext._uri, iocontext._reqparams, checksm, chk_algo); |
| 537 |
} |
| 538 |
|
| 539 |
void HttpMetaOps::makeCollection(IOChainContext & iocontext){ |
| 540 |
internal_make_collection(iocontext._context, iocontext._uri, iocontext._reqparams); |
| 541 |
} |
| 542 |
|
| 543 |
void HttpMetaOps::move(IOChainContext & iocontext, const std::string & target_url){ |
| 544 |
internal_move(iocontext._context, iocontext._uri, iocontext._reqparams, target_url); |
| 545 |
} |
| 546 |
|
| 547 |
void HttpMetaOps::deleteResource(IOChainContext & iocontext){ |
| 548 |
internal_delete_resource(iocontext._context, iocontext._uri, iocontext._reqparams); |
| 549 |
} |
| 550 |
|
| 551 |
StatInfo & HttpMetaOps::statInfo(IOChainContext & iocontext, StatInfo &st_info){ |
| 552 |
struct stat st; |
| 553 |
memset(&st, 0, sizeof(struct stat)); |
| 554 |
getStatInfo(iocontext._context, iocontext._uri, iocontext._reqparams, st_info); |
| 555 |
return st_info; |
| 556 |
} |
| 557 |
|
| 558 |
QuotaInfo & HttpMetaOps::quotaInfo(IOChainContext & iocontext, QuotaInfo &info) { |
| 559 |
getQuotaInfo(iocontext._context, iocontext._uri, iocontext._reqparams, info); |
| 560 |
return info; |
| 561 |
} |
| 562 |
|
| 563 |
bool HttpMetaOps::nextSubItem(IOChainContext &iocontext, std::string &entry_name, StatInfo &info){ |
| 564 |
return webdav_directory_listing(directoryItem, iocontext._context, iocontext._reqparams, iocontext._uri, stat_listing, |
| 565 |
entry_name, info); |
| 566 |
} |
| 567 |
|
| 568 |
///////////////////////// |
| 569 |
///////////////////////// |
| 570 |
|
| 571 |
SwiftMetaOps::SwiftMetaOps() : HttpIOChain() |
| 572 |
{} |
| 573 |
|
| 574 |
SwiftMetaOps::~SwiftMetaOps(){} |
| 575 |
|
| 576 |
static bool is_swift_operation(IOChainContext & context){ |
| 577 |
return context._reqparams->getProtocol() == RequestProtocol::Swift; |
| 578 |
} |
| 579 |
|
| 580 |
static void swiftStatMapper(Context& context, const RequestParams* params, const Uri & uri, struct StatInfo& st_info) { |
| 581 |
const std::string scope = "Davix::swiftStatMapper"; |
| 582 |
DavixError * tmp_err=NULL; |
| 583 |
HeadRequest req(context, uri, &tmp_err); |
| 584 |
|
| 585 |
// we need to modify it, hence copy |
| 586 |
RequestParams p(params); |
| 587 |
|
| 588 |
if(tmp_err == NULL) { |
| 589 |
req.setParameters(p); |
| 590 |
req.executeRequest(&tmp_err); |
| 591 |
const int code = req.getRequestCode(); |
| 592 |
|
| 593 |
if(code == 404) { |
| 594 |
DavixError::clearError(&tmp_err); |
| 595 |
// try to "list" target resource and see if there is anything inside it, if there is, then it's a directory |
| 596 |
Uri new_url = Swift::swiftUriTransformer(uri, p, true); |
| 597 |
|
| 598 |
GetRequest http_req(context, new_url, &tmp_err); |
| 599 |
|
| 600 |
http_req.setParameters(p); |
| 601 |
|
| 602 |
http_req.beginRequest(&tmp_err); |
| 603 |
checkDavixError(&tmp_err); |
| 604 |
|
| 605 |
check_file_status(http_req, scope); |
| 606 |
|
| 607 |
// check response text, if there is data, then it is a directory, otherwise not a directory or file |
| 608 |
char buffer[256+1]; |
| 609 |
const dav_ssize_t ret = http_req.readSegment(buffer, 256, &tmp_err); |
| 610 |
checkDavixError(&tmp_err); |
| 611 |
if(ret == 0){ |
| 612 |
throw DavixException(scope, StatusCode::FileNotFound, "Not a file or directory"); |
| 613 |
} |
| 614 |
else if (ret < 0) { |
| 615 |
throw DavixException(scope, StatusCode::UnknownError, "Unknown readSegment error"); |
| 616 |
} |
| 617 |
checkDavixError(&tmp_err); |
| 618 |
|
| 619 |
st_info.mode = 0755; |
| 620 |
st_info.mode |= S_IFDIR; |
| 621 |
} |
| 622 |
else if(code == 200){ |
| 623 |
st_info.mode = 0755; |
| 624 |
|
| 625 |
std::string swift_path = Swift::extract_swift_path(uri); |
| 626 |
if(swift_path == "/") // is container |
| 627 |
st_info.mode |= S_IFDIR; |
| 628 |
else if(swift_path[swift_path.size()-1] == '/' && req.getAnswerSize() == 0) { // is a directory |
| 629 |
st_info.mode |= S_IFDIR; |
| 630 |
} |
| 631 |
else { // is file |
| 632 |
st_info.mode |= S_IFREG; |
| 633 |
const dav_ssize_t s = req.getAnswerSize(); |
| 634 |
st_info.size = std::max<dav_ssize_t>(0, s); |
| 635 |
st_info.mtime = req.getLastModified(); |
| 636 |
} |
| 637 |
} |
| 638 |
else if(code == 204){ // Normal response of a HEAD request to a container is 204 |
| 639 |
st_info.mode = 0755; |
| 640 |
|
| 641 |
std::string swift_path = Swift::extract_swift_path(uri); |
| 642 |
if(swift_path == "/") // is container |
| 643 |
st_info.mode |= S_IFDIR; |
| 644 |
} |
| 645 |
else if(code == 500){ |
| 646 |
throw DavixException(scope, StatusCode::UnknownError, "Internal Server Error triggered while attempting to get Swift object's stats"); |
| 647 |
} |
| 648 |
} |
| 649 |
checkDavixError(&tmp_err); |
| 650 |
} |
| 651 |
|
| 652 |
StatInfo & SwiftMetaOps::statInfo(IOChainContext &iocontext, StatInfo &st_info) { |
| 653 |
if(is_swift_operation(iocontext)){ |
| 654 |
swiftStatMapper(iocontext._context, iocontext._reqparams, iocontext._uri, st_info); |
| 655 |
return st_info; |
| 656 |
} |
| 657 |
else{ |
| 658 |
StatInfo & ref = HttpIOChain::statInfo(iocontext, st_info); |
| 659 |
return ref; |
| 660 |
} |
| 661 |
} |
| 662 |
|
| 663 |
bool is_a_container(Uri u) { |
| 664 |
std::string tmp = Swift::extract_swift_path(u); |
| 665 |
if(tmp.compare("/") == 0){ |
| 666 |
return true; |
| 667 |
} |
| 668 |
return false; |
| 669 |
} |
| 670 |
|
| 671 |
bool s3_get_next_property(std::unique_ptr<DirHandle> & handle, std::string & name_entry, StatInfo & info); |
| 672 |
|
| 673 |
void swift_start_listing_query(std::unique_ptr<DirHandle> & handle, Context & context, const RequestParams* params, const Uri & url, const std::string & body){ |
| 674 |
(void) body; |
| 675 |
dav_ssize_t s_resu; |
| 676 |
DavixError* tmp_err=NULL; |
| 677 |
|
| 678 |
if(params->getSwiftListingMode() == SwiftListingMode::Hierarchical){ |
| 679 |
Uri new_url = Swift::swiftUriTransformer(url, params, true); |
| 680 |
handle.reset(new DirHandle(new GetRequest(context, new_url, &tmp_err), new SwiftPropParser(Swift::extract_swift_path(url)))); |
| 681 |
} |
| 682 |
else if(params->getSwiftListingMode() == SwiftListingMode::SemiHierarchical){ |
| 683 |
Uri new_url = Swift::swiftUriTransformer(url, params, false); |
| 684 |
handle.reset(new DirHandle(new GetRequest(context, new_url, &tmp_err), new SwiftPropParser(Swift::extract_swift_path(url)))); |
| 685 |
} |
| 686 |
else{ |
| 687 |
if(is_a_container(url) == false){ |
| 688 |
throw DavixException(davix_scope_directory_listing_str(), StatusCode::IsNotADirectory, "This is not a Swift container"); |
| 689 |
} |
| 690 |
handle.reset(new DirHandle(new GetRequest(context, url, &tmp_err), new SwiftPropParser())); |
| 691 |
} |
| 692 |
checkDavixError(&tmp_err); |
| 693 |
|
| 694 |
|
| 695 |
const int operation_timeout = params->getOperationTimeout()->tv_sec; |
| 696 |
HttpRequest & http_req = *(handle->request); |
| 697 |
XMLPropParser & parser = *(handle->parser); |
| 698 |
|
| 699 |
time_t timestamp_timeout = time(NULL) + ((operation_timeout)?(operation_timeout):180); |
| 700 |
|
| 701 |
http_req.setParameters(params); |
| 702 |
http_req.addHeaderField("Accept", "application/xml"); |
| 703 |
|
| 704 |
http_req.beginRequest(&tmp_err); |
| 705 |
checkDavixError(&tmp_err); |
| 706 |
|
| 707 |
check_file_status(http_req, davix_scope_directory_listing_str()); |
| 708 |
|
| 709 |
size_t prop_size = 0; |
| 710 |
do{ // first entry -> container information |
| 711 |
s_resu = incremental_listdir_parsing(&http_req, &parser, 2048, davix_scope_directory_listing_str()); |
| 712 |
|
| 713 |
prop_size = parser.getProperties().size(); |
| 714 |
if(s_resu < 2048 && prop_size <1){ // verify request status : if req done + no data -> error |
| 715 |
throw DavixException(davix_scope_directory_listing_str(), StatusCode::ParsingError, "Invalid server response, not a Swift listing or the directory is empty"); |
| 716 |
} |
| 717 |
if(timestamp_timeout < time(NULL)){ |
| 718 |
throw DavixException(davix_scope_directory_listing_str(), StatusCode::OperationTimeout, "Operation timeout triggered while directory listing"); |
| 719 |
} |
| 720 |
|
| 721 |
}while( prop_size < 1); // prop < 1 means not enough data |
| 722 |
|
| 723 |
} |
| 724 |
|
| 725 |
bool swift_directory_listing(std::unique_ptr<DirHandle> & handle, Context & context, const RequestParams* params, const Uri & uri, const std::string & body, std::string & name_entry, StatInfo & info){ |
| 726 |
if(handle.get() == NULL){ |
| 727 |
swift_start_listing_query(handle, context, params, uri, body); |
| 728 |
} |
| 729 |
return s3_get_next_property(handle, name_entry, info); |
| 730 |
} |
| 731 |
|
| 732 |
|
| 733 |
bool SwiftMetaOps::nextSubItem(IOChainContext &iocontext, std::string &entry_name, StatInfo &info){ |
| 734 |
if(is_swift_operation(iocontext)){ |
| 735 |
return swift_directory_listing(directoryItem, iocontext._context, iocontext._reqparams, iocontext._uri, stat_listing, |
| 736 |
entry_name, info); |
| 737 |
}else{ |
| 738 |
return HttpIOChain::nextSubItem(iocontext, entry_name, info); |
| 739 |
} |
| 740 |
|
| 741 |
} |
| 742 |
|
| 743 |
void SwiftMetaOps::move(IOChainContext & iocontext, const std::string & target_url) { |
| 744 |
const std::string scope = "Davix::SwiftMetaOps::move"; |
| 745 |
if(!is_swift_operation(iocontext)) { |
| 746 |
return HttpIOChain::move(iocontext, target_url); |
| 747 |
} |
| 748 |
|
| 749 |
Context context = iocontext._context; |
| 750 |
RequestParams params = iocontext._reqparams; |
| 751 |
Uri uri(iocontext._uri); |
| 752 |
Uri target(target_url); |
| 753 |
|
| 754 |
// verify both are using the same swift provider/server, reuse s3 utils |
| 755 |
std::string p1 = S3::extract_s3_provider(uri); |
| 756 |
std::string p2 = S3::extract_s3_provider(target); |
| 757 |
|
| 758 |
if(p1 != p2) { |
| 759 |
throw DavixException(scope, StatusCode::OperationNonSupported, |
| 760 |
"It looks that the two URLs are not using the same Swift provider. Unable to perform the move operation."); |
| 761 |
} |
| 762 |
|
| 763 |
std::string source_container = Swift::extract_swift_container(uri); |
| 764 |
std::string source_path = Swift::extract_swift_path(uri); |
| 765 |
|
| 766 |
DavixError *tmp_err = NULL; |
| 767 |
PutRequest req(context, target, &tmp_err); |
| 768 |
checkDavixError(&tmp_err); |
| 769 |
req.setParameters(iocontext._reqparams); |
| 770 |
req.addHeaderField("X-Copy-From", "/" + source_container + source_path); |
| 771 |
|
| 772 |
req.executeRequest(&tmp_err); |
| 773 |
checkDavixError(&tmp_err); |
| 774 |
|
| 775 |
// if copying was successful, delete the source file |
| 776 |
if(req.getRequestCode() == 201) { |
| 777 |
|
| 778 |
DeleteRequest req(context, uri, &tmp_err); |
| 779 |
checkDavixError(&tmp_err); |
| 780 |
|
| 781 |
RequestParams p(iocontext._reqparams); |
| 782 |
req.setParameters(p); |
| 783 |
|
| 784 |
req.executeRequest(&tmp_err); |
| 785 |
checkDavixError(&tmp_err); |
| 786 |
} |
| 787 |
else { |
| 788 |
std::stringstream str; |
| 789 |
str << "Received code " << req.getRequestCode() << " when trying to copy file - will not perform deletion"; |
| 790 |
throw DavixException(scope, StatusCode::UnknownError, str.str()); |
| 791 |
} |
| 792 |
} |
| 793 |
|
| 794 |
///////////////////////// |
| 795 |
///////////////////////// |
| 796 |
|
| 797 |
|
| 798 |
bool is_a_bucket(const Uri & u){ |
| 799 |
const std::string & s = u.getPath(); |
| 800 |
return (std::find_if(s.begin(), s.end(), std::not1(StrUtil::isSlash())) == s.end()); // false if pathname does not match '\/+' |
| 801 |
} |
| 802 |
|
| 803 |
|
| 804 |
|
| 805 |
S3MetaOps::S3MetaOps() : HttpIOChain() |
| 806 |
{} |
| 807 |
|
| 808 |
S3MetaOps::~S3MetaOps(){} |
| 809 |
|
| 810 |
static bool is_s3_operation(IOChainContext & context){ |
| 811 |
const std::string & proto = context._uri.getProtocol(); |
| 812 |
const RequestProtocol::Protocol protocol_flag = context._reqparams->getProtocol(); |
| 813 |
|
| 814 |
if( proto.compare(0, 2, "s3") ==0 || protocol_flag == RequestProtocol::AwsS3) return true; |
| 815 |
if( proto.compare(0, 6, "gcloud") ==0 || protocol_flag == RequestProtocol::Gcloud) return true; |
| 816 |
|
| 817 |
return false; |
| 818 |
} |
| 819 |
|
| 820 |
static void internal_s3_create_bucket_or_dir(Context & c, const Uri & url, const RequestParams & params){ |
| 821 |
DavixError * tmp_err=NULL; |
| 822 |
|
| 823 |
/* make sure path ends with a slash, otherwise s3 |
| 824 |
will just create a zero-length file */ |
| 825 |
Uri url2 = url; |
| 826 |
if(url.getPath()[url.getPath().size()-1] != '/') { |
| 827 |
url2.setPath(url.getPath() + "/"); |
| 828 |
} |
| 829 |
|
| 830 |
PutRequest req(c, url2, &tmp_err); |
| 831 |
req.addHeaderField("Content-Length", "0"); |
| 832 |
checkDavixError(&tmp_err); |
| 833 |
|
| 834 |
req.setParameters(params); |
| 835 |
if( req.executeRequest(&tmp_err) < 0){ |
| 836 |
const int code = req.getRequestCode(); |
| 837 |
httpcodeToDavixException(code, davix_scope_meta(), "bucket creation failure"); |
| 838 |
} |
| 839 |
|
| 840 |
checkDavixError(&tmp_err); |
| 841 |
} |
| 842 |
|
| 843 |
void S3MetaOps::move(IOChainContext & iocontext, const std::string & target_url) { |
| 844 |
const std::string scope = "Davix::S3MetaOps::move"; |
| 845 |
if(!is_s3_operation(iocontext)) { |
| 846 |
return HttpIOChain::move(iocontext, target_url); |
| 847 |
} |
| 848 |
|
| 849 |
Context context = iocontext._context; |
| 850 |
RequestParams params = iocontext._reqparams; |
| 851 |
Uri uri(iocontext._uri); |
| 852 |
Uri target(target_url); |
| 853 |
|
| 854 |
// verify both are using the same s3 provider/server |
| 855 |
std::string p1 = S3::extract_s3_provider(uri); |
| 856 |
std::string p2 = S3::extract_s3_provider(target); |
| 857 |
|
| 858 |
if(p1 != p2) { |
| 859 |
throw DavixException(scope, StatusCode::OperationNonSupported, |
| 860 |
"It looks that the two URLs are not using the same S3 provider. Unable to perform the move operation."); |
| 861 |
} |
| 862 |
|
| 863 |
std::string source_bucket = S3::extract_s3_bucket(uri, params.getAwsAlternate()); |
| 864 |
std::string source_path = S3::extract_s3_path(uri, params.getAwsAlternate()); |
| 865 |
|
| 866 |
DavixError *tmp_err = NULL; |
| 867 |
PutRequest req(context, target, &tmp_err); |
| 868 |
checkDavixError(&tmp_err); |
| 869 |
req.setParameters(iocontext._reqparams); |
| 870 |
req.addHeaderField("x-amz-copy-source", "/" + source_bucket + source_path); |
| 871 |
|
| 872 |
req.executeRequest(&tmp_err); |
| 873 |
checkDavixError(&tmp_err); |
| 874 |
|
| 875 |
// if copying was successful, delete the source file |
| 876 |
if(req.getRequestCode() == 200) { |
| 877 |
std::string region = S3::detect_region(uri); |
| 878 |
DAVIX_SLOG(DAVIX_LOG_DEBUG, DAVIX_LOG_CHAIN, "Detected region for source endpoint: " + region); |
| 879 |
checkDavixError(&tmp_err); |
| 880 |
|
| 881 |
DeleteRequest req(context, uri, &tmp_err); |
| 882 |
checkDavixError(&tmp_err); |
| 883 |
|
| 884 |
RequestParams p(iocontext._reqparams); |
| 885 |
p.setAwsRegion(region); |
| 886 |
req.setParameters(p); |
| 887 |
|
| 888 |
req.executeRequest(&tmp_err); |
| 889 |
checkDavixError(&tmp_err); |
| 890 |
} |
| 891 |
else { |
| 892 |
std::stringstream str; |
| 893 |
str << "Received code " << req.getRequestCode() << " when trying to copy file - will not perform deletion"; |
| 894 |
throw DavixException(scope, StatusCode::UnknownError, str.str()); |
| 895 |
} |
| 896 |
} |
| 897 |
|
| 898 |
|
| 899 |
void S3MetaOps::checksum(IOChainContext &iocontext, std::string &checksm, const std::string &chk_algo){ |
| 900 |
internal_checksum(iocontext._context, iocontext._uri, iocontext._reqparams, checksm, chk_algo); |
| 901 |
} |
| 902 |
|
| 903 |
void S3MetaOps::makeCollection(IOChainContext &iocontext){ |
| 904 |
if(is_s3_operation(iocontext) || is_swift_operation(iocontext)){ |
| 905 |
internal_s3_create_bucket_or_dir( iocontext._context, iocontext._uri, iocontext._reqparams); |
| 906 |
}else{ |
| 907 |
HttpIOChain::makeCollection(iocontext); |
| 908 |
} |
| 909 |
} |
| 910 |
|
| 911 |
|
| 912 |
void s3StatMapper(Context& context, const RequestParams* params, const Uri & uri, struct StatInfo& st_info){ |
| 913 |
const std::string scope = "Davix::s3StatMapper"; |
| 914 |
DavixError * tmp_err=NULL; |
| 915 |
HeadRequest req(context, uri, &tmp_err); |
| 916 |
|
| 917 |
// we need to modify it, hence copy |
| 918 |
RequestParams p(params); |
| 919 |
// we just need to know if target has anything inside it |
| 920 |
p.setS3MaxKey(1); |
| 921 |
|
| 922 |
if( tmp_err == NULL){ |
| 923 |
req.setParameters(p); |
| 924 |
req.executeRequest(&tmp_err); |
| 925 |
const int code = req.getRequestCode(); |
| 926 |
|
| 927 |
// if 404, target either doesn't exist or is a S3 "directory" |
| 928 |
if(code == 404){ |
| 929 |
DavixError::clearError(&tmp_err); |
| 930 |
// try to "list" target resource and see if there is anything inside it, if there is, then it's a directory |
| 931 |
Uri new_url = S3::s3UriTransformer(uri, p, true); |
| 932 |
DirHandle handle(new GetRequest(context, new_url, &tmp_err), new S3PropParser(params->getS3ListingMode(), S3::extract_s3_path(uri, params->getAwsAlternate()))); |
| 933 |
|
| 934 |
dav_ssize_t s_resu=0; |
| 935 |
|
| 936 |
const int operation_timeout = p.getOperationTimeout()->tv_sec; |
| 937 |
HttpRequest & http_req = *(handle.request); |
| 938 |
XMLPropParser & parser = *(handle.parser); |
| 939 |
|
| 940 |
time_t timestamp_timeout = time(NULL) + ((operation_timeout)?(operation_timeout):180); |
| 941 |
|
| 942 |
http_req.setParameters(p); |
| 943 |
|
| 944 |
http_req.beginRequest(&tmp_err); |
| 945 |
checkDavixError(&tmp_err); |
| 946 |
|
| 947 |
check_file_status(http_req, scope); |
| 948 |
|
| 949 |
size_t prop_size = 0; |
| 950 |
do{ // first entry |
| 951 |
TRY_DAVIX{ |
| 952 |
s_resu = incremental_listdir_parsing(&http_req, &parser, 2048, scope); |
| 953 |
}CATCH_DAVIX(&tmp_err) |
| 954 |
|
| 955 |
if(tmp_err && (tmp_err->getStatus() == StatusCode::IsNotADirectory)){ |
| 956 |
std::ostringstream ss; |
| 957 |
ss << uri << " not found"; |
| 958 |
throw DavixException(scope, StatusCode::FileNotFound, ss.str()); |
| 959 |
} |
| 960 |
|
| 961 |
prop_size = parser.getProperties().size(); |
| 962 |
if(s_resu < 2048 && prop_size <1){ // verify request status : if req done + no data -> error |
| 963 |
throw DavixException(scope, StatusCode::ParsingError, "Invalid server response, not a S3 listing"); |
| 964 |
} |
| 965 |
if(timestamp_timeout < time(NULL)){ |
| 966 |
throw DavixException(scope, StatusCode::OperationTimeout, "Operation timeout triggered while getting S3 object's stats"); |
| 967 |
} |
| 968 |
|
| 969 |
}while( prop_size < 1); // prop < 1 means not enough data |
| 970 |
|
| 971 |
st_info.mode = 0755; |
| 972 |
st_info.mode |= S_IFDIR; |
| 973 |
} |
| 974 |
else if(code == 200){ // found something, must be a file not directory |
| 975 |
st_info.mode = 0755; |
| 976 |
|
| 977 |
std::string s3_path = S3::extract_s3_path(uri, params->getAwsAlternate()); |
| 978 |
if(s3_path == "/") // is bucket |
| 979 |
st_info.mode |= S_IFDIR; |
| 980 |
else if(s3_path[s3_path.size()-1] == '/' && req.getAnswerSize() == 0) { // is a directory |
| 981 |
st_info.mode |= S_IFDIR; |
| 982 |
} |
| 983 |
else{ // is file |
| 984 |
st_info.mode |= S_IFREG; |
| 985 |
const dav_ssize_t s = req.getAnswerSize(); |
| 986 |
st_info.size = std::max<dav_ssize_t>(0,s); |
| 987 |
st_info.mtime = req.getLastModified(); |
| 988 |
} |
| 989 |
} |
| 990 |
else if(code == 500) |
| 991 |
throw DavixException(scope, StatusCode::UnknownError, "Internal Server Error triggered while attempting to get S3 object's stats"); |
| 992 |
} |
| 993 |
checkDavixError(&tmp_err); |
| 994 |
} |
| 995 |
|
| 996 |
|
| 997 |
// get statInfo |
| 998 |
StatInfo & S3MetaOps::statInfo(IOChainContext & iocontext, StatInfo & st_info){ |
| 999 |
if(is_s3_operation(iocontext)){ |
| 1000 |
s3StatMapper(iocontext._context, iocontext._reqparams, iocontext._uri, st_info); |
| 1001 |
return st_info; |
| 1002 |
} |
| 1003 |
else{ |
| 1004 |
StatInfo & ref = HttpIOChain::statInfo(iocontext, st_info); |
| 1005 |
return ref; |
| 1006 |
} |
| 1007 |
} |
| 1008 |
|
| 1009 |
|
| 1010 |
bool s3_get_next_property(std::unique_ptr<DirHandle> & handle, std::string & name_entry, StatInfo & info){ |
| 1011 |
DAVIX_SLOG(DAVIX_LOG_DEBUG, DAVIX_LOG_CHAIN, " -> s3_get_next_property"); |
| 1012 |
const size_t read_size = 2048*1024; |
| 1013 |
|
| 1014 |
|
| 1015 |
HttpRequest& req = *(handle->request); // setup env again |
| 1016 |
XMLPropParser& parser = *(handle->parser); |
| 1017 |
|
| 1018 |
size_t prop_size = parser.getProperties().size(); |
| 1019 |
ssize_t s_resu = read_size; |
| 1020 |
|
| 1021 |
while( prop_size == 0 |
| 1022 |
&& s_resu > 0){ // execute request only if no property are available |
| 1023 |
|
| 1024 |
// continue the parsing until one more result |
| 1025 |
s_resu = incremental_listdir_parsing(&req, &parser, read_size, "S3::listing"); |
| 1026 |
prop_size = parser.getProperties().size(); |
| 1027 |
} |
| 1028 |
|
| 1029 |
|
| 1030 |
if(prop_size == 0){ |
| 1031 |
return false; // end of the request, end of the story |
| 1032 |
} |
| 1033 |
|
| 1034 |
FileProperties & front = parser.getProperties().front(); |
| 1035 |
name_entry.swap(front.filename); |
| 1036 |
info = front.info; |
| 1037 |
parser.getProperties().pop_front(); // clean the current element |
| 1038 |
return true; |
| 1039 |
} |
| 1040 |
|
| 1041 |
|
| 1042 |
void s3_start_listing_query(std::unique_ptr<DirHandle> & handle, Context & context, const RequestParams* params, const Uri & url, const std::string & body){ |
| 1043 |
(void) body; |
| 1044 |
dav_ssize_t s_resu; |
| 1045 |
DavixError* tmp_err=NULL; |
| 1046 |
bool listing_buckets; |
| 1047 |
|
| 1048 |
if(params->getProtocol() == RequestProtocol::Gcloud) { |
| 1049 |
Uri new_url = gcloud::getListingURI(url, params); |
| 1050 |
|
| 1051 |
std::string prefix = gcloud::extract_path(url); |
| 1052 |
if(prefix != "/") prefix = "/" + prefix; |
| 1053 |
handle.reset(new DirHandle(new GetRequest(context, new_url, &tmp_err), new S3PropParser(params->getS3ListingMode(), prefix))); |
| 1054 |
} |
| 1055 |
else if(params->getS3ListingMode() == S3ListingMode::Hierarchical){ |
| 1056 |
Uri new_url = S3::s3UriTransformer(url, params, true); |
| 1057 |
if (handle.get() != NULL) { |
| 1058 |
XMLPropParser & parser = *(handle->parser); |
| 1059 |
std::string nextmarker = parser.getNextMarker(); |
| 1060 |
if (nextmarker != "") { |
| 1061 |
new_url.addQueryParam("marker", nextmarker); |
| 1062 |
} |
| 1063 |
} |
| 1064 |
handle.reset(new DirHandle(new GetRequest(context, new_url, &tmp_err), new S3PropParser(params->getS3ListingMode(), S3::extract_s3_path(url, params->getAwsAlternate())))); |
| 1065 |
} |
| 1066 |
else if(params->getS3ListingMode() == S3ListingMode::SemiHierarchical){ |
| 1067 |
Uri new_url = S3::s3UriTransformer(url, params, false); |
| 1068 |
handle.reset(new DirHandle(new GetRequest(context, new_url, &tmp_err), new S3PropParser(params->getS3ListingMode(), S3::extract_s3_path(url, params->getAwsAlternate())))); |
| 1069 |
} |
| 1070 |
else{ |
| 1071 |
if(is_a_bucket(url) == false){ |
| 1072 |
throw DavixException(davix_scope_directory_listing_str(), StatusCode::IsNotADirectory, "This is not a S3 bucket"); |
| 1073 |
} |
| 1074 |
handle.reset(new DirHandle(new GetRequest(context, url, &tmp_err), new S3PropParser())); |
| 1075 |
} |
| 1076 |
checkDavixError(&tmp_err); |
| 1077 |
|
| 1078 |
// Check if we are listing available buckets |
| 1079 |
listing_buckets = (params->getAwsAlternate() && url.getPath() == "/"); |
| 1080 |
|
| 1081 |
|
| 1082 |
const int operation_timeout = params->getOperationTimeout()->tv_sec; |
| 1083 |
HttpRequest & http_req = *(handle->request); |
| 1084 |
XMLPropParser & parser = *(handle->parser); |
| 1085 |
|
| 1086 |
time_t timestamp_timeout = time(NULL) + ((operation_timeout)?(operation_timeout):180); |
| 1087 |
|
| 1088 |
http_req.setParameters(params); |
| 1089 |
|
| 1090 |
http_req.beginRequest(&tmp_err); |
| 1091 |
checkDavixError(&tmp_err); |
| 1092 |
|
| 1093 |
check_file_status(http_req, davix_scope_directory_listing_str()); |
| 1094 |
|
| 1095 |
size_t prop_size = 0; |
| 1096 |
do{ // first entry -> bucket information |
| 1097 |
s_resu = incremental_listdir_parsing(&http_req, &parser, 2048*1024, davix_scope_directory_listing_str()); |
| 1098 |
|
| 1099 |
prop_size = parser.getProperties().size(); |
| 1100 |
if(s_resu < 2048*1024 && prop_size <1){ // verify request status : if req done + no data -> error |
| 1101 |
throw DavixException(davix_scope_directory_listing_str(), StatusCode::ParsingError, "Invalid server response, not a S3 listing"); |
| 1102 |
} |
| 1103 |
if(timestamp_timeout < time(NULL)){ |
| 1104 |
throw DavixException(davix_scope_directory_listing_str(), StatusCode::OperationTimeout, "Operation timeout triggered while directory listing"); |
| 1105 |
} |
| 1106 |
|
| 1107 |
}while( prop_size < 1); // prop < 1 means not enough data |
| 1108 |
|
| 1109 |
const StatInfo & info = parser.getProperties().at(0).info; |
| 1110 |
if( S_ISDIR(info.mode) == false){ |
| 1111 |
std::ostringstream ss; |
| 1112 |
ss << url << " is not a S3 bucket"; |
| 1113 |
throw DavixException(davix_scope_directory_listing_str(), StatusCode::IsNotADirectory, ss.str()); |
| 1114 |
} else if (!listing_buckets) { |
| 1115 |
parser.getProperties().pop_front(); // suppress the bucket name entry |
| 1116 |
} |
| 1117 |
|
| 1118 |
} |
| 1119 |
|
| 1120 |
|
| 1121 |
|
| 1122 |
bool s3_directory_listing(std::unique_ptr<DirHandle> & handle, Context & context, const RequestParams* params, const Uri & uri, const std::string & body, std::string & name_entry, StatInfo & info){ |
| 1123 |
if (handle.get() == NULL){ |
| 1124 |
s3_start_listing_query(handle, context, params, uri, body); |
| 1125 |
} else { |
| 1126 |
XMLPropParser& parser = *(handle->parser); |
| 1127 |
if (parser.getProperties().empty() && !parser.getNextMarker().empty()) { |
| 1128 |
s3_start_listing_query(handle, context, params, uri, body); |
| 1129 |
} |
| 1130 |
} |
| 1131 |
return s3_get_next_property(handle, name_entry, info); |
| 1132 |
} |
| 1133 |
|
| 1134 |
|
| 1135 |
bool S3MetaOps::nextSubItem(IOChainContext &iocontext, std::string &entry_name, StatInfo &info){ |
| 1136 |
if(is_s3_operation(iocontext)){ |
| 1137 |
return s3_directory_listing(directoryItem, iocontext._context, iocontext._reqparams, iocontext._uri, stat_listing, |
| 1138 |
entry_name, info); |
| 1139 |
}else{ |
| 1140 |
return HttpIOChain::nextSubItem(iocontext, entry_name, info); |
| 1141 |
} |
| 1142 |
|
| 1143 |
} |
| 1144 |
|
| 1145 |
///////////////////////// |
| 1146 |
///////////////////////// |
| 1147 |
|
| 1148 |
AzureMetaOps::AzureMetaOps() : HttpIOChain() |
| 1149 |
{} |
| 1150 |
|
| 1151 |
AzureMetaOps::~AzureMetaOps(){} |
| 1152 |
|
| 1153 |
static bool is_azure_operation(IOChainContext & context){ |
| 1154 |
return context._reqparams->getProtocol() == RequestProtocol::Azure; |
| 1155 |
} |
| 1156 |
|
| 1157 |
void azureStatMapper(Context& context, const RequestParams* params, const Uri & uri, struct StatInfo& st_info) { |
| 1158 |
const std::string scope = "Davix::azureStatMapper"; |
| 1159 |
DavixError * tmp_err=NULL; |
| 1160 |
|
| 1161 |
//Context c; |
| 1162 |
HeadRequest req(context, uri, &tmp_err); |
| 1163 |
|
| 1164 |
// we need to modify it, hence copy |
| 1165 |
RequestParams p(params); |
| 1166 |
// we just need to know if target has anything inside it |
| 1167 |
//p.setS3MaxKey(1); |
| 1168 |
|
| 1169 |
if( tmp_err == NULL){ |
| 1170 |
req.setParameters(p); |
| 1171 |
req.executeRequest(&tmp_err); |
| 1172 |
const int code = req.getRequestCode(); |
| 1173 |
|
| 1174 |
// if 404, target either doesn't exist or is an Azure "directory". TODO: add support to stat directories. must have func tests by then |
| 1175 |
if(code == 404){ |
| 1176 |
DavixError::clearError(&tmp_err); |
| 1177 |
Uri new_url = Azure::transformURI(uri, p, true); |
| 1178 |
DirHandle handle(new GetRequest(context, new_url, &tmp_err), new AzurePropParser(Azure::extract_azure_filename(uri))); |
| 1179 |
|
| 1180 |
dav_ssize_t s_resu=0; |
| 1181 |
|
| 1182 |
const int operation_timeout = p.getOperationTimeout()->tv_sec; |
| 1183 |
HttpRequest & http_req = *(handle.request); |
| 1184 |
XMLPropParser & parser = *(handle.parser); |
| 1185 |
|
| 1186 |
time_t timestamp_timeout = time(NULL) + ((operation_timeout)?(operation_timeout):180); |
| 1187 |
|
| 1188 |
http_req.setParameters(p); |
| 1189 |
|
| 1190 |
http_req.beginRequest(&tmp_err); |
| 1191 |
checkDavixError(&tmp_err); |
| 1192 |
check_file_status(http_req, scope); |
| 1193 |
|
| 1194 |
size_t prop_size = 0; |
| 1195 |
do{ // first entry -> container information |
| 1196 |
s_resu = incremental_listdir_parsing(&http_req, &parser, 2048, davix_scope_directory_listing_str()); |
| 1197 |
|
| 1198 |
prop_size = parser.getProperties().size(); |
| 1199 |
if(s_resu < 2048 && prop_size <1){ // verify request status : if req done + no data -> error |
| 1200 |
throw DavixException(davix_scope_directory_listing_str(), StatusCode::IsNotADirectory, "The specified directory does not exist"); |
| 1201 |
} |
| 1202 |
if(timestamp_timeout < time(NULL)){ |
| 1203 |
throw DavixException(davix_scope_directory_listing_str(), StatusCode::OperationTimeout, "Operation timeout triggered while directory listing"); |
| 1204 |
} |
| 1205 |
}while( prop_size < 1); // prop < 1 means not enough data |
| 1206 |
|
| 1207 |
st_info.mode = 0755; |
| 1208 |
st_info.mode |= S_IFDIR; |
| 1209 |
} |
| 1210 |
// file exists, parse its info |
| 1211 |
else if(code == 200) { |
| 1212 |
st_info.mode = 0755; |
| 1213 |
st_info.mode |= S_IFREG; |
| 1214 |
const dav_ssize_t s = req.getAnswerSize(); |
| 1215 |
st_info.size = std::max<dav_ssize_t>(0,s); |
| 1216 |
st_info.mtime = req.getLastModified(); |
| 1217 |
} |
| 1218 |
|
| 1219 |
} |
| 1220 |
|
| 1221 |
} |
| 1222 |
|
| 1223 |
StatInfo & AzureMetaOps::statInfo(IOChainContext & iocontext, StatInfo & st_info) { |
| 1224 |
if(is_azure_operation(iocontext)) { |
| 1225 |
azureStatMapper(iocontext._context, iocontext._reqparams, iocontext._uri, st_info); |
| 1226 |
return st_info; |
| 1227 |
} |
| 1228 |
else { |
| 1229 |
return HttpIOChain::statInfo(iocontext, st_info); |
| 1230 |
} |
| 1231 |
} |
| 1232 |
|
| 1233 |
static void azure_start_listing_query(std::unique_ptr<DirHandle> & handle, Context & context, const RequestParams* params, const Uri & url, const std::string & body) { |
| 1234 |
DavixError* tmp_err=NULL; |
| 1235 |
dav_ssize_t s_resu; |
| 1236 |
|
| 1237 |
Uri new_url = Davix::Azure::transformURI(url, params, true); |
| 1238 |
handle.reset(new DirHandle(new GetRequest(context, new_url, &tmp_err), new AzurePropParser(Davix::Azure::extract_azure_filename(url)))); |
| 1239 |
|
| 1240 |
const int operation_timeout = params->getOperationTimeout()->tv_sec; |
| 1241 |
HttpRequest & http_req = *(handle->request); |
| 1242 |
XMLPropParser & parser = *(handle->parser); |
| 1243 |
time_t timestamp_timeout = time(NULL) + ((operation_timeout)?(operation_timeout):180); |
| 1244 |
|
| 1245 |
http_req.setParameters(params); |
| 1246 |
|
| 1247 |
http_req.beginRequest(&tmp_err); |
| 1248 |
checkDavixError(&tmp_err); |
| 1249 |
|
| 1250 |
check_file_status(http_req, davix_scope_directory_listing_str()); |
| 1251 |
|
| 1252 |
size_t prop_size = 0; |
| 1253 |
do{ // first entry -> container information |
| 1254 |
s_resu = incremental_listdir_parsing(&http_req, &parser, 2048, davix_scope_directory_listing_str()); |
| 1255 |
|
| 1256 |
prop_size = parser.getProperties().size(); |
| 1257 |
if(s_resu < 2048 && prop_size <1){ // verify request status : if req done + no data -> error |
| 1258 |
throw DavixException(davix_scope_directory_listing_str(), StatusCode::IsNotADirectory, "The specified directory does not exist"); |
| 1259 |
} |
| 1260 |
if(timestamp_timeout < time(NULL)){ |
| 1261 |
throw DavixException(davix_scope_directory_listing_str(), StatusCode::OperationTimeout, "Operation timeout triggered while directory listing"); |
| 1262 |
} |
| 1263 |
}while( prop_size < 1); // prop < 1 means not enough data |
| 1264 |
|
| 1265 |
} |
| 1266 |
|
| 1267 |
bool azure_get_next_property(std::unique_ptr<DirHandle> & handle, std::string & name_entry, StatInfo & info) { |
| 1268 |
DAVIX_SLOG(DAVIX_LOG_DEBUG, DAVIX_LOG_CHAIN, " -> azure_get_next_property"); |
| 1269 |
const size_t read_size = 2048; |
| 1270 |
|
| 1271 |
HttpRequest& req = *(handle->request); // setup env again |
| 1272 |
XMLPropParser& parser = *(handle->parser); |
| 1273 |
|
| 1274 |
size_t prop_size = parser.getProperties().size(); |
| 1275 |
ssize_t s_resu = read_size; |
| 1276 |
|
| 1277 |
while( prop_size == 0 |
| 1278 |
&& s_resu > 0){ // execute request only if no property are available |
| 1279 |
|
| 1280 |
// continue the parsing until one more result |
| 1281 |
s_resu = incremental_listdir_parsing(&req, &parser, read_size, "S3::listing"); |
| 1282 |
prop_size = parser.getProperties().size(); |
| 1283 |
} |
| 1284 |
|
| 1285 |
if(prop_size == 0){ |
| 1286 |
return false; // end of the request, end of the story |
| 1287 |
} |
| 1288 |
|
| 1289 |
FileProperties & front = parser.getProperties().front(); |
| 1290 |
name_entry.swap(front.filename); |
| 1291 |
info = front.info; |
| 1292 |
parser.getProperties().pop_front(); // clean the current element |
| 1293 |
return true; |
| 1294 |
} |
| 1295 |
|
| 1296 |
static bool azure_directory_listing(std::unique_ptr<DirHandle> & handle, Context & context, const RequestParams* params, const Uri & uri, const std::string & body, std::string & name_entry, StatInfo & info){ |
| 1297 |
if(handle.get() == NULL){ |
| 1298 |
azure_start_listing_query(handle, context, params, uri, body); |
| 1299 |
} |
| 1300 |
return azure_get_next_property(handle, name_entry, info); |
| 1301 |
} |
| 1302 |
|
| 1303 |
bool AzureMetaOps::nextSubItem(IOChainContext &iocontext, std::string &entry_name, StatInfo &info) { |
| 1304 |
if(is_azure_operation(iocontext)){ |
| 1305 |
return azure_directory_listing(directoryItem, iocontext._context, iocontext._reqparams, iocontext._uri, stat_listing, |
| 1306 |
entry_name, info); |
| 1307 |
}else{ |
| 1308 |
return HttpIOChain::nextSubItem(iocontext, entry_name, info); |
| 1309 |
} |
| 1310 |
} |
| 1311 |
|
| 1312 |
|
| 1313 |
} // Davix |