Debian Codegraph @

src/fileops/davmeta.cpp

raw
1 /*
2 * This File is part of Davix, The IO library for HTTP based protocols
3 * Copyright (C) CERN 2013
4 * Author: Adrien Devresse <[email protected]>
5 *
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2.1 of the License, or (at your option) any later version.
10 *
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, write to the Free Software
18 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19 *
20 */
21
22 #include <davix_internal.hpp>
23 #include "davmeta.hpp"
24
25 #include <xml/davpropxmlparser.hpp>
26 #include <xml/davdeletexmlparser.hpp>
27 #include <xml/metalinkparser.hpp>
28 #include <xml/s3propparser.hpp>
29 #include <xml/azurepropparser.hpp>
30 #include <xml/swiftpropparser.hpp>
31
32 #include <utils/davix_logger_internal.hpp>
33 #include <utils/davix_utils_internal.hpp>
34 #include <utils/davix_s3_utils.hpp>
35 #include <utils/davix_azure_utils.hpp>
36 #include <utils/davix_gcloud_utils.hpp>
37 #include <utils/davix_swift_utils.hpp>
38 #include <utils/checksum_extractor.hpp>
39
40 #include <request/httprequest.hpp>
41 #include <fileops/fileutils.hpp>
42 #include <utils/stringutils.hpp>
43 #include "libs/alibxx/crypto/base64.hpp"
44 #include <neon/neonrequest.hpp>
45
46
47 using namespace StrUtil;
48
49 namespace Davix{
50
51 static const std::string stat_listing("<?xml version=\"1.0\" encoding=\"utf-8\" ?><D:propfind xmlns:D=\"DAV:\" xmlns:L=\"LCGDM:\"><D:prop>"
52 "<D:displayname/><D:getlastmodified/><D:creationdate/><D:getcontentlength/><D:quota-used-bytes/>"
53 "<D:resourcetype><D:collection/></D:resourcetype><L:mode/>"
54 "<D:owner></D:owner><D:group></D:group>"
55 "</D:prop>"
56 "</D:propfind>");
57
58 static const std::string quota_stat("<?xml version=\"1.0\" encoding=\"utf-8\" ?><D:propfind xmlns:D=\"DAV:\" xmlns:L=\"LCGDM:\"><D:prop>"
59 "<D:quota-used-bytes/><D:quota-available-bytes/>"
60 "</D:prop>"
61 "</D:propfind>");
62
63 struct DirHandle{
64
65 DirHandle(HttpRequest* req, XMLPropParser * p): request(req), parser(p){}
66
67 std::unique_ptr<HttpRequest> request;
68 std::unique_ptr<Davix::XMLPropParser> parser;
69
70
71 };
72
73 /**
74 * Helper function to execute a PROPFIND/stat request on a given HTTP request handle.
75 * Returns a string with the HTTP response.
76 */
77 std::string req_webdav_propfind(HttpRequest* req, DavixError** err) {
78 req->addHeaderField("Depth", "0");
79 req->setRequestMethod("PROPFIND");
80
81 if (req->executeRequest(err) == 0) {
82 auto content = req->getAnswerContent();
83
84 if (content != NULL) {
85 return {content};
86 }
87 }
88
89 return {};
90 }
91
92
93 int dav_stat_mapper_webdav(Context &context, const RequestParams* params, const Uri & url, struct StatInfo& st_info){
94 DavPropXMLParser parser;
95 DavixError* tmp_err = NULL;
96 HttpRequest req(context, url, &tmp_err);
97 int ret = -1;
98
99 if( tmp_err == NULL){
100 req.setParameters(params);
101
102 TRY_DAVIX {
103 std::string response = req_webdav_propfind(&req, &tmp_err);
104
105 if (!tmp_err) {
106 parser.parseChunk(response);
107 std::deque<FileProperties>& props = parser.getProperties();
108
109 if (props.size() < 1) {
110 throw DavixException(davix_scope_stat_str(), Davix::StatusCode::WebDavPropertiesParsingError, "Parsing Error: properties number < 1");
111 } else {
112 st_info = props.front().info;
113 ret = 0;
114 }
115 }
116 } CATCH_DAVIX(&tmp_err)
117
118 if (tmp_err != NULL) {
119 ret = -1;
120 }
121 }
122
123 checkDavixError(&tmp_err);
124 return ret;
125 }
126
127
128 int dav_stat_mapper_http(Context& context, const RequestParams* params, const Uri & uri, struct StatInfo& st_info){
129 int ret = -1;
130 DavixError * tmp_err=NULL;
131 HeadRequest req(context, uri, &tmp_err);
132
133 if( tmp_err == NULL){
134 req.setParameters(params);
135 req.executeRequest(&tmp_err);
136
137 if(!tmp_err){
138 if(httpcodeIsValid(req.getRequestCode()) ){
139 memset(&st_info, 0, sizeof(struct StatInfo));
140 const dav_ssize_t s = req.getAnswerSize();
141 st_info.size = std::max<dav_ssize_t>(0,s);
142 st_info.mode = 0755 | S_IFREG;
143 ret = 0;
144 }else{
145 httpcodeToDavixError(req.getRequestCode(), davix_scope_http_request(), uri.getString() , &tmp_err);
146 ret = -1;
147 }
148 }
149 }
150 checkDavixError(&tmp_err);
151 return ret;
152 }
153
154
155 // Implement stat with a GET of Range 1
156 int dav_stat_mapper_http_get(Context& context, const RequestParams* params, const Uri & uri, struct StatInfo& st_info){
157 int ret = -1;
158 DavixError * tmp_err=NULL;
159 GetRequest req(context, uri, &tmp_err);
160
161 if( tmp_err == NULL){
162 req.setParameters(params);
163 req.addHeaderField("Range", "bytes=0-1");
164 req.executeRequest(&tmp_err);
165
166 if(!tmp_err){
167 if(httpcodeIsValid(req.getRequestCode()) ){
168 memset(&st_info, 0, sizeof(struct StatInfo));
169 std::string rnge;
170 req.getAnswerHeader("Content-Range", rnge);
171 std::string::size_type pos = rnge.find_first_of("/");
172 if (pos == std::string::npos) {
173 throw DavixException(davix_scope_meta(), StatusCode::ParsingError, "Content-Range not parsable");
174 }
175 if (rnge.substr(pos+1,1) == "*") {
176 throw DavixException(davix_scope_meta(), StatusCode::ParsingError, "Server does not provide content length");
177 }
178 long lsize = toType<long, std::string>()(rnge.substr(pos+1));
179 st_info.size = std::max<long>(0,lsize);
180 st_info.mode = 0755 | S_IFREG;
181 req.discardBody(&tmp_err);
182 ret = 0;
183 }else{
184 httpcodeToDavixError(req.getRequestCode(), davix_scope_http_request(), uri.getString() , &tmp_err);
185 ret = -1;
186 }
187 }
188 }
189 checkDavixError(&tmp_err);
190 return ret;
191 }
192
193
194 dav_ssize_t incremental_listdir_parsing(HttpRequest* req, XMLPropParser * parser, dav_size_t s_buff, const std::string & scope){
195 DavixError* tmp_err=NULL;
196
197 char buffer[s_buff+1];
198 const dav_ssize_t ret = req->readSegment(buffer, s_buff, &tmp_err);
199 checkDavixError(&tmp_err);
200 if(ret >= 0){
201 buffer[ret]= '\0';
202 parser->parseChunk(buffer, ret);
203 }else{
204 throw DavixException(scope, StatusCode::UnknownError, "Unknown readSegment error");
205 }
206
207 return ret;
208 }
209
210
211 dav_ssize_t getStatInfo(Context & c, const Uri & url, const RequestParams * p,
212 struct StatInfo& st_info){
213 RequestParams params(p);
214 configureRequestParamsProto(url, params);
215 int ret =-1;
216
217 switch(params.getProtocol()){
218 case RequestProtocol::Webdav:
219 ret = dav_stat_mapper_webdav(c, &params, url, st_info);
220 break;
221 default:
222 if (isS3SignedURL(url)) {
223 // This endpoint won't accept a HEAD request, use GET instead
224 ret = dav_stat_mapper_http_get(c, &params, url, st_info);
225 } else {
226 ret = dav_stat_mapper_http(c, &params, url, st_info);
227 }
228 break;
229
230 }
231 DAVIX_SLOG(DAVIX_LOG_DEBUG, DAVIX_LOG_CHAIN, " davix_stat <-");
232 return ret;
233 }
234
235 QuotaInfo::QuotaInfo() : d_ptr(new Internal()) { }
236 QuotaInfo::~QuotaInfo() { }
237 dav_size_t QuotaInfo::getUsedBytes() {
238 return d_ptr->used_bytes;
239 }
240 dav_size_t QuotaInfo::getFreeSpace() {
241 return d_ptr->free_space;
242 }
243
244 class QuotaInfoHandler {
245 public:
246 static void setdptr(QuotaInfo &info, QuotaInfo::Internal &internal) {
247 info.d_ptr.reset(new QuotaInfo::Internal(internal));
248 }
249 };
250
251 void getQuotaInfo(Context & c, const Uri & url, const RequestParams *p, QuotaInfo &info) {
252 DavixError * tmp_err = NULL;
253 HttpRequest req(c, url, &tmp_err);
254 checkDavixError(&tmp_err);
255
256 req.setParameters(*p);
257 req.addHeaderField("Depth","0");
258 req.setRequestMethod("PROPFIND");
259 req.setRequestBody(quota_stat);
260
261 if (req.executeRequest(&tmp_err) == 0 && !tmp_err) {
262 DavPropXMLParser parser;
263 auto content = req.getAnswerContent();
264 std::string response = (content != NULL) ? content : "";
265 parser.parseChunk(response);
266 std::deque<FileProperties> & props = parser.getProperties();
267
268 if (props.size() < 1) {
269 throw DavixException(davix_scope_stat_str(), Davix::StatusCode::WebDavPropertiesParsingError, "Parsing Error: properties number < 1");
270 } else {
271 QuotaInfoHandler::setdptr(info, props.front().quota);
272 }
273 }
274
275 checkDavixError(&tmp_err);
276 }
277
278 void parse_creation_deletion_result(int code, const Uri & u, const std::string & scope, const std::vector<char> & body){
279 switch(code) {
280 case 200:
281 case 201:
282 case 202:
283 case 204: {
284 return;
285 }
286 case 207: {
287 // Parse WebDAV
288 DavDeleteXMLParser parser;
289 std::string response = (!body.empty()) ? body.data() : "";
290 parser.parseChunk(response);
291
292 if (parser.getProperties().size() > 0) {
293 for (unsigned int i = 0; i < parser.getProperties().size(); ++i) {
294 const int sub_code = parser.getProperties().at(i).req_status;
295 std::ostringstream ss;
296 ss << "occurred during deletion request for " << parser.getProperties().at(i).filename;
297
298 if (httpcodeIsValid(sub_code) == false) {
299 httpcodeToDavixException(sub_code, scope, ss.str());
300 }
301 }
302
303 return;
304 }
305
306 // if no properties, properties were filtered because invalid
307 httpcodeToDavixException(404, scope);
308 break;
309 }
310 }
311
312 std::ostringstream ss;
313 ss << " with url " << u.getString();
314 httpcodeToDavixException(code, scope, ss.str());
315 }
316
317
318 int internal_delete_resource(Context & c, const Uri & url, const RequestParams & params){
319 DavixError* tmp_err=NULL;
320 int ret=-1;
321 RequestParams _params(params);
322
323 DeleteRequest req(c, url, &tmp_err);
324 req.setParameters(_params);
325 if(!tmp_err){
326 if( ( ret=req.executeRequest(&tmp_err)) == 0){
327 parse_creation_deletion_result(req.getRequestCode(), url, davix_scope_rm_str(), req.getAnswerContentVec());
328 }
329 }
330
331
332 checkDavixError(&tmp_err);
333 return ret;
334 }
335
336
337 int internal_make_collection(Context & c, const Uri & url, const RequestParams & params){
338 DAVIX_SCOPE_TRACE(DAVIX_LOG_CHAIN, mkcoll);
339
340 int ret=-1;
341 DavixError* tmp_err=NULL;
342 RequestParams _params(params);
343
344 HttpRequest req(c, url, &tmp_err);
345
346 if(tmp_err == NULL){
347 req.setParameters(params);
348 req.setRequestMethod("MKCOL");
349 if( (ret = req.executeRequest(&tmp_err)) == 0){
350 parse_creation_deletion_result(req.getRequestCode(), url, davix_scope_mkdir_str(), req.getAnswerContentVec());
351 }
352
353 }
354
355 checkDavixError(&tmp_err);
356 return ret;
357 }
358
359
360 int internal_move(Context & c, const Uri & url, const RequestParams & params, const std::string & target_url){
361 DAVIX_SLOG(DAVIX_LOG_DEBUG, DAVIX_LOG_CHAIN, " -> move");
362 int ret=-1;
363 DavixError* tmp_err=NULL;
364 RequestParams _params(params);
365
366 HttpRequest req(c, url, &tmp_err);
367
368 if(tmp_err == NULL){
369 req.setParameters(params);
370 req.setRequestMethod("MOVE");
371
372 Uri uri(target_url);
373 uri.httpizeProtocol();
374 req.addHeaderField("Destination", uri.getString());
375
376 if( (ret = req.executeRequest(&tmp_err)) == 0){
377 parse_creation_deletion_result(req.getRequestCode(), url, davix_scope_mv_str(), req.getAnswerContentVec());
378 }
379
380 }
381
382 DAVIX_SLOG(DAVIX_LOG_DEBUG, DAVIX_LOG_CHAIN, " move <-");
383 checkDavixError(&tmp_err);
384 return ret;
385 }
386
387
388
389 int internal_checksum(Context & c, const Uri & url, const RequestParams *p, std::string & checksm, const std::string & chk_algo){
390 DAVIX_SLOG(DAVIX_LOG_DEBUG, DAVIX_LOG_CHAIN, " -> checksum");
391 int ret=-1;
392 DavixError* tmp_err=NULL;
393 RequestParams params(p);
394
395 HeadRequest req(c, url, &tmp_err);
396
397 if(tmp_err == NULL){
398 // add Digest file, support for other digest, extended format
399 req.addHeaderField("Want-Digest", chk_algo);
400 req.setParameters(params);
401 if( (ret = req.executeRequest(&tmp_err)) == 0
402 && !tmp_err && (ret = davixRequestToFileStatus(&req, davix_scope_mkdir_str(), &tmp_err)) >=0){
403
404 // try simple MD5 ( standard )
405 if(compare_ncase(chk_algo, "MD5") == 0){
406 std::string chk;
407 if(req.getAnswerHeader("Content-MD5", chk) == true){
408 DAVIX_SLOG(DAVIX_LOG_TRACE, DAVIX_LOG_CHAIN, "Extract MD5 checksum in base64 {}", chk);
409 chk= Base64::base64_decode(chk);
410 std::swap(checksm, chk);
411 return 0;
412 }
413 }
414
415 // fallback on extension for checksum
416 HeaderVec headers;
417 req.getAnswerHeaders(headers);
418
419 if(ChecksumExtractor::extractChecksum(headers, chk_algo, checksm)) {
420 return 0;
421 }
422
423 // last chance try to extract MD5 checksum from ETAG ( S3 work around )
424 std::string etag_str;
425 if(compare_ncase(chk_algo, "MD5") ==0 && req.getAnswerHeader("etag", etag_str)){
426 stringVec tokens = tokenSplit(etag_str, "&;\\/\"'");
427 for(stringVec::iterator it = tokens.begin(); it < tokens.end(); it++){
428 if(it->size() == 32 && std::find_if(it->begin(), it->end(), std::not1(StrUtil::isHexa())) == it->end()){
429 std::swap(checksm, *it);
430 return 0;
431 }
432 }
433 }
434
435 std::ostringstream ss;
436 ss << "checksum calculation for " << chk_algo << " not supported for " << url;
437 throw DavixException(davix_scope_meta(), StatusCode::OperationNonSupported, ss.str());
438
439 DAVIX_SLOG(DAVIX_LOG_DEBUG, DAVIX_LOG_CHAIN, " checksum <-");
440 return 0;
441 }
442 }
443 throw DavixException(&tmp_err);
444 }
445
446
447
448
449 bool wedav_get_next_property(std::unique_ptr<DirHandle> & handle, std::string & name_entry, StatInfo & info){
450 DAVIX_SLOG(DAVIX_LOG_DEBUG, DAVIX_LOG_CHAIN, " -> wedav_get_next_property");
451 const size_t read_size = 2048;
452
453
454 HttpRequest& req = *(handle->request); // setup env again
455 XMLPropParser& parser = *(handle->parser);
456
457 size_t prop_size = parser.getProperties().size();
458 ssize_t s_resu = read_size;
459
460 while( prop_size == 0
461 && s_resu > 0){ // request not complete and current data too smalls
462 // continue the parsing until one more result
463 s_resu = incremental_listdir_parsing(&req, &parser, read_size, "WebDav::listing");
464
465 prop_size = parser.getProperties().size();
466 }
467
468
469 if(prop_size == 0){
470 return false; // end of the request, end of the story
471 }
472
473 FileProperties & front = parser.getProperties().front();
474 name_entry.swap(front.filename);
475 info = front.info;
476 parser.getProperties().pop_front(); // clean the current element
477 return true;
478 }
479
480
481 void webdav_start_listing_query(std::unique_ptr<DirHandle> & handle, Context & context, const RequestParams* params, const Uri & url, const std::string & body){
482 dav_ssize_t s_resu;
483
484 DavixError* tmp_err=NULL;
485 handle.reset(new DirHandle(new PropfindRequest(context, url, &tmp_err), new DavPropXMLParser()));
486 checkDavixError(&tmp_err);
487
488 HttpRequest & http_req = *(handle->request);
489 XMLPropParser & parser = *(handle->parser);
490
491 http_req.addHeaderField("Depth","1");
492 http_req.setParameters(params);
493 // setup the handle for simple listing only
494 http_req.setRequestBody(body);
495
496 http_req.beginRequest(&tmp_err);
497 checkDavixError(&tmp_err);
498
499 check_file_status(http_req, davix_scope_directory_listing_str());
500
501 size_t prop_size = 0;
502 do{ // parse the begining of the request until the first property -> directory property
503 s_resu = incremental_listdir_parsing(&http_req, &parser, 2048, davix_scope_directory_listing_str());
504
505 prop_size = parser.getProperties().size();
506 if(s_resu < 2048 && prop_size <1){ // verify request status : if req done + no data -> error
507 throw DavixException(davix_scope_directory_listing_str(), StatusCode::WebDavPropertiesParsingError, "bad server answer, not a valid WebDav PROPFIND answer");
508 }
509
510 }while( prop_size < 1); // leave is end of req & no data
511
512 const StatInfo & info = parser.getProperties().at(0).info;
513 if( S_ISDIR(info.mode) == false){
514 std::ostringstream ss;
515 ss << url << " is not a collection, listing impossible";
516 throw DavixException(davix_scope_directory_listing_str(), StatusCode::IsNotADirectory, ss.str());
517 }else{
518 parser.getProperties().pop_front(); // suppress the parent directory infos...
519 }
520
521 }
522
523 bool webdav_directory_listing(std::unique_ptr<DirHandle> & handle, Context & context, const RequestParams* params, const Uri & uri, const std::string & body, std::string & name_entry, StatInfo & info){
524 if(handle.get() == NULL){
525 webdav_start_listing_query(handle, context, params, uri, body);
526 }
527 return wedav_get_next_property(handle, name_entry, info);
528 }
529
530 HttpMetaOps::HttpMetaOps(): HttpIOChain(){}
531
532 HttpMetaOps::~HttpMetaOps(){}
533
534
535 void HttpMetaOps::checksum(IOChainContext & iocontext, std::string &checksm, const std::string &chk_algo){
536 internal_checksum(iocontext._context, iocontext._uri, iocontext._reqparams, checksm, chk_algo);
537 }
538
539 void HttpMetaOps::makeCollection(IOChainContext & iocontext){
540 internal_make_collection(iocontext._context, iocontext._uri, iocontext._reqparams);
541 }
542
543 void HttpMetaOps::move(IOChainContext & iocontext, const std::string & target_url){
544 internal_move(iocontext._context, iocontext._uri, iocontext._reqparams, target_url);
545 }
546
547 void HttpMetaOps::deleteResource(IOChainContext & iocontext){
548 internal_delete_resource(iocontext._context, iocontext._uri, iocontext._reqparams);
549 }
550
551 StatInfo & HttpMetaOps::statInfo(IOChainContext & iocontext, StatInfo &st_info){
552 struct stat st;
553 memset(&st, 0, sizeof(struct stat));
554 getStatInfo(iocontext._context, iocontext._uri, iocontext._reqparams, st_info);
555 return st_info;
556 }
557
558 QuotaInfo & HttpMetaOps::quotaInfo(IOChainContext & iocontext, QuotaInfo &info) {
559 getQuotaInfo(iocontext._context, iocontext._uri, iocontext._reqparams, info);
560 return info;
561 }
562
563 bool HttpMetaOps::nextSubItem(IOChainContext &iocontext, std::string &entry_name, StatInfo &info){
564 return webdav_directory_listing(directoryItem, iocontext._context, iocontext._reqparams, iocontext._uri, stat_listing,
565 entry_name, info);
566 }
567
568 /////////////////////////
569 /////////////////////////
570
571 SwiftMetaOps::SwiftMetaOps() : HttpIOChain()
572 {}
573
574 SwiftMetaOps::~SwiftMetaOps(){}
575
576 static bool is_swift_operation(IOChainContext & context){
577 return context._reqparams->getProtocol() == RequestProtocol::Swift;
578 }
579
580 static void swiftStatMapper(Context& context, const RequestParams* params, const Uri & uri, struct StatInfo& st_info) {
581 const std::string scope = "Davix::swiftStatMapper";
582 DavixError * tmp_err=NULL;
583 HeadRequest req(context, uri, &tmp_err);
584
585 // we need to modify it, hence copy
586 RequestParams p(params);
587
588 if(tmp_err == NULL) {
589 req.setParameters(p);
590 req.executeRequest(&tmp_err);
591 const int code = req.getRequestCode();
592
593 if(code == 404) {
594 DavixError::clearError(&tmp_err);
595 // try to "list" target resource and see if there is anything inside it, if there is, then it's a directory
596 Uri new_url = Swift::swiftUriTransformer(uri, p, true);
597
598 GetRequest http_req(context, new_url, &tmp_err);
599
600 http_req.setParameters(p);
601
602 http_req.beginRequest(&tmp_err);
603 checkDavixError(&tmp_err);
604
605 check_file_status(http_req, scope);
606
607 // check response text, if there is data, then it is a directory, otherwise not a directory or file
608 char buffer[256+1];
609 const dav_ssize_t ret = http_req.readSegment(buffer, 256, &tmp_err);
610 checkDavixError(&tmp_err);
611 if(ret == 0){
612 throw DavixException(scope, StatusCode::FileNotFound, "Not a file or directory");
613 }
614 else if (ret < 0) {
615 throw DavixException(scope, StatusCode::UnknownError, "Unknown readSegment error");
616 }
617 checkDavixError(&tmp_err);
618
619 st_info.mode = 0755;
620 st_info.mode |= S_IFDIR;
621 }
622 else if(code == 200){
623 st_info.mode = 0755;
624
625 std::string swift_path = Swift::extract_swift_path(uri);
626 if(swift_path == "/") // is container
627 st_info.mode |= S_IFDIR;
628 else if(swift_path[swift_path.size()-1] == '/' && req.getAnswerSize() == 0) { // is a directory
629 st_info.mode |= S_IFDIR;
630 }
631 else { // is file
632 st_info.mode |= S_IFREG;
633 const dav_ssize_t s = req.getAnswerSize();
634 st_info.size = std::max<dav_ssize_t>(0, s);
635 st_info.mtime = req.getLastModified();
636 }
637 }
638 else if(code == 204){ // Normal response of a HEAD request to a container is 204
639 st_info.mode = 0755;
640
641 std::string swift_path = Swift::extract_swift_path(uri);
642 if(swift_path == "/") // is container
643 st_info.mode |= S_IFDIR;
644 }
645 else if(code == 500){
646 throw DavixException(scope, StatusCode::UnknownError, "Internal Server Error triggered while attempting to get Swift object's stats");
647 }
648 }
649 checkDavixError(&tmp_err);
650 }
651
652 StatInfo & SwiftMetaOps::statInfo(IOChainContext &iocontext, StatInfo &st_info) {
653 if(is_swift_operation(iocontext)){
654 swiftStatMapper(iocontext._context, iocontext._reqparams, iocontext._uri, st_info);
655 return st_info;
656 }
657 else{
658 StatInfo & ref = HttpIOChain::statInfo(iocontext, st_info);
659 return ref;
660 }
661 }
662
663 bool is_a_container(Uri u) {
664 std::string tmp = Swift::extract_swift_path(u);
665 if(tmp.compare("/") == 0){
666 return true;
667 }
668 return false;
669 }
670
671 bool s3_get_next_property(std::unique_ptr<DirHandle> & handle, std::string & name_entry, StatInfo & info);
672
673 void swift_start_listing_query(std::unique_ptr<DirHandle> & handle, Context & context, const RequestParams* params, const Uri & url, const std::string & body){
674 (void) body;
675 dav_ssize_t s_resu;
676 DavixError* tmp_err=NULL;
677
678 if(params->getSwiftListingMode() == SwiftListingMode::Hierarchical){
679 Uri new_url = Swift::swiftUriTransformer(url, params, true);
680 handle.reset(new DirHandle(new GetRequest(context, new_url, &tmp_err), new SwiftPropParser(Swift::extract_swift_path(url))));
681 }
682 else if(params->getSwiftListingMode() == SwiftListingMode::SemiHierarchical){
683 Uri new_url = Swift::swiftUriTransformer(url, params, false);
684 handle.reset(new DirHandle(new GetRequest(context, new_url, &tmp_err), new SwiftPropParser(Swift::extract_swift_path(url))));
685 }
686 else{
687 if(is_a_container(url) == false){
688 throw DavixException(davix_scope_directory_listing_str(), StatusCode::IsNotADirectory, "This is not a Swift container");
689 }
690 handle.reset(new DirHandle(new GetRequest(context, url, &tmp_err), new SwiftPropParser()));
691 }
692 checkDavixError(&tmp_err);
693
694
695 const int operation_timeout = params->getOperationTimeout()->tv_sec;
696 HttpRequest & http_req = *(handle->request);
697 XMLPropParser & parser = *(handle->parser);
698
699 time_t timestamp_timeout = time(NULL) + ((operation_timeout)?(operation_timeout):180);
700
701 http_req.setParameters(params);
702 http_req.addHeaderField("Accept", "application/xml");
703
704 http_req.beginRequest(&tmp_err);
705 checkDavixError(&tmp_err);
706
707 check_file_status(http_req, davix_scope_directory_listing_str());
708
709 size_t prop_size = 0;
710 do{ // first entry -> container information
711 s_resu = incremental_listdir_parsing(&http_req, &parser, 2048, davix_scope_directory_listing_str());
712
713 prop_size = parser.getProperties().size();
714 if(s_resu < 2048 && prop_size <1){ // verify request status : if req done + no data -> error
715 throw DavixException(davix_scope_directory_listing_str(), StatusCode::ParsingError, "Invalid server response, not a Swift listing or the directory is empty");
716 }
717 if(timestamp_timeout < time(NULL)){
718 throw DavixException(davix_scope_directory_listing_str(), StatusCode::OperationTimeout, "Operation timeout triggered while directory listing");
719 }
720
721 }while( prop_size < 1); // prop < 1 means not enough data
722
723 }
724
725 bool swift_directory_listing(std::unique_ptr<DirHandle> & handle, Context & context, const RequestParams* params, const Uri & uri, const std::string & body, std::string & name_entry, StatInfo & info){
726 if(handle.get() == NULL){
727 swift_start_listing_query(handle, context, params, uri, body);
728 }
729 return s3_get_next_property(handle, name_entry, info);
730 }
731
732
733 bool SwiftMetaOps::nextSubItem(IOChainContext &iocontext, std::string &entry_name, StatInfo &info){
734 if(is_swift_operation(iocontext)){
735 return swift_directory_listing(directoryItem, iocontext._context, iocontext._reqparams, iocontext._uri, stat_listing,
736 entry_name, info);
737 }else{
738 return HttpIOChain::nextSubItem(iocontext, entry_name, info);
739 }
740
741 }
742
743 void SwiftMetaOps::move(IOChainContext & iocontext, const std::string & target_url) {
744 const std::string scope = "Davix::SwiftMetaOps::move";
745 if(!is_swift_operation(iocontext)) {
746 return HttpIOChain::move(iocontext, target_url);
747 }
748
749 Context context = iocontext._context;
750 RequestParams params = iocontext._reqparams;
751 Uri uri(iocontext._uri);
752 Uri target(target_url);
753
754 // verify both are using the same swift provider/server, reuse s3 utils
755 std::string p1 = S3::extract_s3_provider(uri);
756 std::string p2 = S3::extract_s3_provider(target);
757
758 if(p1 != p2) {
759 throw DavixException(scope, StatusCode::OperationNonSupported,
760 "It looks that the two URLs are not using the same Swift provider. Unable to perform the move operation.");
761 }
762
763 std::string source_container = Swift::extract_swift_container(uri);
764 std::string source_path = Swift::extract_swift_path(uri);
765
766 DavixError *tmp_err = NULL;
767 PutRequest req(context, target, &tmp_err);
768 checkDavixError(&tmp_err);
769 req.setParameters(iocontext._reqparams);
770 req.addHeaderField("X-Copy-From", "/" + source_container + source_path);
771
772 req.executeRequest(&tmp_err);
773 checkDavixError(&tmp_err);
774
775 // if copying was successful, delete the source file
776 if(req.getRequestCode() == 201) {
777
778 DeleteRequest req(context, uri, &tmp_err);
779 checkDavixError(&tmp_err);
780
781 RequestParams p(iocontext._reqparams);
782 req.setParameters(p);
783
784 req.executeRequest(&tmp_err);
785 checkDavixError(&tmp_err);
786 }
787 else {
788 std::stringstream str;
789 str << "Received code " << req.getRequestCode() << " when trying to copy file - will not perform deletion";
790 throw DavixException(scope, StatusCode::UnknownError, str.str());
791 }
792 }
793
794 /////////////////////////
795 /////////////////////////
796
797
798 bool is_a_bucket(const Uri & u){
799 const std::string & s = u.getPath();
800 return (std::find_if(s.begin(), s.end(), std::not1(StrUtil::isSlash())) == s.end()); // false if pathname does not match '\/+'
801 }
802
803
804
805 S3MetaOps::S3MetaOps() : HttpIOChain()
806 {}
807
808 S3MetaOps::~S3MetaOps(){}
809
810 static bool is_s3_operation(IOChainContext & context){
811 const std::string & proto = context._uri.getProtocol();
812 const RequestProtocol::Protocol protocol_flag = context._reqparams->getProtocol();
813
814 if( proto.compare(0, 2, "s3") ==0 || protocol_flag == RequestProtocol::AwsS3) return true;
815 if( proto.compare(0, 6, "gcloud") ==0 || protocol_flag == RequestProtocol::Gcloud) return true;
816
817 return false;
818 }
819
820 static void internal_s3_create_bucket_or_dir(Context & c, const Uri & url, const RequestParams & params){
821 DavixError * tmp_err=NULL;
822
823 /* make sure path ends with a slash, otherwise s3
824 will just create a zero-length file */
825 Uri url2 = url;
826 if(url.getPath()[url.getPath().size()-1] != '/') {
827 url2.setPath(url.getPath() + "/");
828 }
829
830 PutRequest req(c, url2, &tmp_err);
831 req.addHeaderField("Content-Length", "0");
832 checkDavixError(&tmp_err);
833
834 req.setParameters(params);
835 if( req.executeRequest(&tmp_err) < 0){
836 const int code = req.getRequestCode();
837 httpcodeToDavixException(code, davix_scope_meta(), "bucket creation failure");
838 }
839
840 checkDavixError(&tmp_err);
841 }
842
843 void S3MetaOps::move(IOChainContext & iocontext, const std::string & target_url) {
844 const std::string scope = "Davix::S3MetaOps::move";
845 if(!is_s3_operation(iocontext)) {
846 return HttpIOChain::move(iocontext, target_url);
847 }
848
849 Context context = iocontext._context;
850 RequestParams params = iocontext._reqparams;
851 Uri uri(iocontext._uri);
852 Uri target(target_url);
853
854 // verify both are using the same s3 provider/server
855 std::string p1 = S3::extract_s3_provider(uri);
856 std::string p2 = S3::extract_s3_provider(target);
857
858 if(p1 != p2) {
859 throw DavixException(scope, StatusCode::OperationNonSupported,
860 "It looks that the two URLs are not using the same S3 provider. Unable to perform the move operation.");
861 }
862
863 std::string source_bucket = S3::extract_s3_bucket(uri, params.getAwsAlternate());
864 std::string source_path = S3::extract_s3_path(uri, params.getAwsAlternate());
865
866 DavixError *tmp_err = NULL;
867 PutRequest req(context, target, &tmp_err);
868 checkDavixError(&tmp_err);
869 req.setParameters(iocontext._reqparams);
870 req.addHeaderField("x-amz-copy-source", "/" + source_bucket + source_path);
871
872 req.executeRequest(&tmp_err);
873 checkDavixError(&tmp_err);
874
875 // if copying was successful, delete the source file
876 if(req.getRequestCode() == 200) {
877 std::string region = S3::detect_region(uri);
878 DAVIX_SLOG(DAVIX_LOG_DEBUG, DAVIX_LOG_CHAIN, "Detected region for source endpoint: " + region);
879 checkDavixError(&tmp_err);
880
881 DeleteRequest req(context, uri, &tmp_err);
882 checkDavixError(&tmp_err);
883
884 RequestParams p(iocontext._reqparams);
885 p.setAwsRegion(region);
886 req.setParameters(p);
887
888 req.executeRequest(&tmp_err);
889 checkDavixError(&tmp_err);
890 }
891 else {
892 std::stringstream str;
893 str << "Received code " << req.getRequestCode() << " when trying to copy file - will not perform deletion";
894 throw DavixException(scope, StatusCode::UnknownError, str.str());
895 }
896 }
897
898
899 void S3MetaOps::checksum(IOChainContext &iocontext, std::string &checksm, const std::string &chk_algo){
900 internal_checksum(iocontext._context, iocontext._uri, iocontext._reqparams, checksm, chk_algo);
901 }
902
903 void S3MetaOps::makeCollection(IOChainContext &iocontext){
904 if(is_s3_operation(iocontext) || is_swift_operation(iocontext)){
905 internal_s3_create_bucket_or_dir( iocontext._context, iocontext._uri, iocontext._reqparams);
906 }else{
907 HttpIOChain::makeCollection(iocontext);
908 }
909 }
910
911
912 void s3StatMapper(Context& context, const RequestParams* params, const Uri & uri, struct StatInfo& st_info){
913 const std::string scope = "Davix::s3StatMapper";
914 DavixError * tmp_err=NULL;
915 HeadRequest req(context, uri, &tmp_err);
916
917 // we need to modify it, hence copy
918 RequestParams p(params);
919 // we just need to know if target has anything inside it
920 p.setS3MaxKey(1);
921
922 if( tmp_err == NULL){
923 req.setParameters(p);
924 req.executeRequest(&tmp_err);
925 const int code = req.getRequestCode();
926
927 // if 404, target either doesn't exist or is a S3 "directory"
928 if(code == 404){
929 DavixError::clearError(&tmp_err);
930 // try to "list" target resource and see if there is anything inside it, if there is, then it's a directory
931 Uri new_url = S3::s3UriTransformer(uri, p, true);
932 DirHandle handle(new GetRequest(context, new_url, &tmp_err), new S3PropParser(params->getS3ListingMode(), S3::extract_s3_path(uri, params->getAwsAlternate())));
933
934 dav_ssize_t s_resu=0;
935
936 const int operation_timeout = p.getOperationTimeout()->tv_sec;
937 HttpRequest & http_req = *(handle.request);
938 XMLPropParser & parser = *(handle.parser);
939
940 time_t timestamp_timeout = time(NULL) + ((operation_timeout)?(operation_timeout):180);
941
942 http_req.setParameters(p);
943
944 http_req.beginRequest(&tmp_err);
945 checkDavixError(&tmp_err);
946
947 check_file_status(http_req, scope);
948
949 size_t prop_size = 0;
950 do{ // first entry
951 TRY_DAVIX{
952 s_resu = incremental_listdir_parsing(&http_req, &parser, 2048, scope);
953 }CATCH_DAVIX(&tmp_err)
954
955 if(tmp_err && (tmp_err->getStatus() == StatusCode::IsNotADirectory)){
956 std::ostringstream ss;
957 ss << uri << " not found";
958 throw DavixException(scope, StatusCode::FileNotFound, ss.str());
959 }
960
961 prop_size = parser.getProperties().size();
962 if(s_resu < 2048 && prop_size <1){ // verify request status : if req done + no data -> error
963 throw DavixException(scope, StatusCode::ParsingError, "Invalid server response, not a S3 listing");
964 }
965 if(timestamp_timeout < time(NULL)){
966 throw DavixException(scope, StatusCode::OperationTimeout, "Operation timeout triggered while getting S3 object's stats");
967 }
968
969 }while( prop_size < 1); // prop < 1 means not enough data
970
971 st_info.mode = 0755;
972 st_info.mode |= S_IFDIR;
973 }
974 else if(code == 200){ // found something, must be a file not directory
975 st_info.mode = 0755;
976
977 std::string s3_path = S3::extract_s3_path(uri, params->getAwsAlternate());
978 if(s3_path == "/") // is bucket
979 st_info.mode |= S_IFDIR;
980 else if(s3_path[s3_path.size()-1] == '/' && req.getAnswerSize() == 0) { // is a directory
981 st_info.mode |= S_IFDIR;
982 }
983 else{ // is file
984 st_info.mode |= S_IFREG;
985 const dav_ssize_t s = req.getAnswerSize();
986 st_info.size = std::max<dav_ssize_t>(0,s);
987 st_info.mtime = req.getLastModified();
988 }
989 }
990 else if(code == 500)
991 throw DavixException(scope, StatusCode::UnknownError, "Internal Server Error triggered while attempting to get S3 object's stats");
992 }
993 checkDavixError(&tmp_err);
994 }
995
996
997 // get statInfo
998 StatInfo & S3MetaOps::statInfo(IOChainContext & iocontext, StatInfo & st_info){
999 if(is_s3_operation(iocontext)){
1000 s3StatMapper(iocontext._context, iocontext._reqparams, iocontext._uri, st_info);
1001 return st_info;
1002 }
1003 else{
1004 StatInfo & ref = HttpIOChain::statInfo(iocontext, st_info);
1005 return ref;
1006 }
1007 }
1008
1009
1010 bool s3_get_next_property(std::unique_ptr<DirHandle> & handle, std::string & name_entry, StatInfo & info){
1011 DAVIX_SLOG(DAVIX_LOG_DEBUG, DAVIX_LOG_CHAIN, " -> s3_get_next_property");
1012 const size_t read_size = 2048*1024;
1013
1014
1015 HttpRequest& req = *(handle->request); // setup env again
1016 XMLPropParser& parser = *(handle->parser);
1017
1018 size_t prop_size = parser.getProperties().size();
1019 ssize_t s_resu = read_size;
1020
1021 while( prop_size == 0
1022 && s_resu > 0){ // execute request only if no property are available
1023
1024 // continue the parsing until one more result
1025 s_resu = incremental_listdir_parsing(&req, &parser, read_size, "S3::listing");
1026 prop_size = parser.getProperties().size();
1027 }
1028
1029
1030 if(prop_size == 0){
1031 return false; // end of the request, end of the story
1032 }
1033
1034 FileProperties & front = parser.getProperties().front();
1035 name_entry.swap(front.filename);
1036 info = front.info;
1037 parser.getProperties().pop_front(); // clean the current element
1038 return true;
1039 }
1040
1041
1042 void s3_start_listing_query(std::unique_ptr<DirHandle> & handle, Context & context, const RequestParams* params, const Uri & url, const std::string & body){
1043 (void) body;
1044 dav_ssize_t s_resu;
1045 DavixError* tmp_err=NULL;
1046 bool listing_buckets;
1047
1048 if(params->getProtocol() == RequestProtocol::Gcloud) {
1049 Uri new_url = gcloud::getListingURI(url, params);
1050
1051 std::string prefix = gcloud::extract_path(url);
1052 if(prefix != "/") prefix = "/" + prefix;
1053 handle.reset(new DirHandle(new GetRequest(context, new_url, &tmp_err), new S3PropParser(params->getS3ListingMode(), prefix)));
1054 }
1055 else if(params->getS3ListingMode() == S3ListingMode::Hierarchical){
1056 Uri new_url = S3::s3UriTransformer(url, params, true);
1057 if (handle.get() != NULL) {
1058 XMLPropParser & parser = *(handle->parser);
1059 std::string nextmarker = parser.getNextMarker();
1060 if (nextmarker != "") {
1061 new_url.addQueryParam("marker", nextmarker);
1062 }
1063 }
1064 handle.reset(new DirHandle(new GetRequest(context, new_url, &tmp_err), new S3PropParser(params->getS3ListingMode(), S3::extract_s3_path(url, params->getAwsAlternate()))));
1065 }
1066 else if(params->getS3ListingMode() == S3ListingMode::SemiHierarchical){
1067 Uri new_url = S3::s3UriTransformer(url, params, false);
1068 handle.reset(new DirHandle(new GetRequest(context, new_url, &tmp_err), new S3PropParser(params->getS3ListingMode(), S3::extract_s3_path(url, params->getAwsAlternate()))));
1069 }
1070 else{
1071 if(is_a_bucket(url) == false){
1072 throw DavixException(davix_scope_directory_listing_str(), StatusCode::IsNotADirectory, "This is not a S3 bucket");
1073 }
1074 handle.reset(new DirHandle(new GetRequest(context, url, &tmp_err), new S3PropParser()));
1075 }
1076 checkDavixError(&tmp_err);
1077
1078 // Check if we are listing available buckets
1079 listing_buckets = (params->getAwsAlternate() && url.getPath() == "/");
1080
1081
1082 const int operation_timeout = params->getOperationTimeout()->tv_sec;
1083 HttpRequest & http_req = *(handle->request);
1084 XMLPropParser & parser = *(handle->parser);
1085
1086 time_t timestamp_timeout = time(NULL) + ((operation_timeout)?(operation_timeout):180);
1087
1088 http_req.setParameters(params);
1089
1090 http_req.beginRequest(&tmp_err);
1091 checkDavixError(&tmp_err);
1092
1093 check_file_status(http_req, davix_scope_directory_listing_str());
1094
1095 size_t prop_size = 0;
1096 do{ // first entry -> bucket information
1097 s_resu = incremental_listdir_parsing(&http_req, &parser, 2048*1024, davix_scope_directory_listing_str());
1098
1099 prop_size = parser.getProperties().size();
1100 if(s_resu < 2048*1024 && prop_size <1){ // verify request status : if req done + no data -> error
1101 throw DavixException(davix_scope_directory_listing_str(), StatusCode::ParsingError, "Invalid server response, not a S3 listing");
1102 }
1103 if(timestamp_timeout < time(NULL)){
1104 throw DavixException(davix_scope_directory_listing_str(), StatusCode::OperationTimeout, "Operation timeout triggered while directory listing");
1105 }
1106
1107 }while( prop_size < 1); // prop < 1 means not enough data
1108
1109 const StatInfo & info = parser.getProperties().at(0).info;
1110 if( S_ISDIR(info.mode) == false){
1111 std::ostringstream ss;
1112 ss << url << " is not a S3 bucket";
1113 throw DavixException(davix_scope_directory_listing_str(), StatusCode::IsNotADirectory, ss.str());
1114 } else if (!listing_buckets) {
1115 parser.getProperties().pop_front(); // suppress the bucket name entry
1116 }
1117
1118 }
1119
1120
1121
1122 bool s3_directory_listing(std::unique_ptr<DirHandle> & handle, Context & context, const RequestParams* params, const Uri & uri, const std::string & body, std::string & name_entry, StatInfo & info){
1123 if (handle.get() == NULL){
1124 s3_start_listing_query(handle, context, params, uri, body);
1125 } else {
1126 XMLPropParser& parser = *(handle->parser);
1127 if (parser.getProperties().empty() && !parser.getNextMarker().empty()) {
1128 s3_start_listing_query(handle, context, params, uri, body);
1129 }
1130 }
1131 return s3_get_next_property(handle, name_entry, info);
1132 }
1133
1134
1135 bool S3MetaOps::nextSubItem(IOChainContext &iocontext, std::string &entry_name, StatInfo &info){
1136 if(is_s3_operation(iocontext)){
1137 return s3_directory_listing(directoryItem, iocontext._context, iocontext._reqparams, iocontext._uri, stat_listing,
1138 entry_name, info);
1139 }else{
1140 return HttpIOChain::nextSubItem(iocontext, entry_name, info);
1141 }
1142
1143 }
1144
1145 /////////////////////////
1146 /////////////////////////
1147
1148 AzureMetaOps::AzureMetaOps() : HttpIOChain()
1149 {}
1150
1151 AzureMetaOps::~AzureMetaOps(){}
1152
1153 static bool is_azure_operation(IOChainContext & context){
1154 return context._reqparams->getProtocol() == RequestProtocol::Azure;
1155 }
1156
1157 void azureStatMapper(Context& context, const RequestParams* params, const Uri & uri, struct StatInfo& st_info) {
1158 const std::string scope = "Davix::azureStatMapper";
1159 DavixError * tmp_err=NULL;
1160
1161 //Context c;
1162 HeadRequest req(context, uri, &tmp_err);
1163
1164 // we need to modify it, hence copy
1165 RequestParams p(params);
1166 // we just need to know if target has anything inside it
1167 //p.setS3MaxKey(1);
1168
1169 if( tmp_err == NULL){
1170 req.setParameters(p);
1171 req.executeRequest(&tmp_err);
1172 const int code = req.getRequestCode();
1173
1174 // if 404, target either doesn't exist or is an Azure "directory". TODO: add support to stat directories. must have func tests by then
1175 if(code == 404){
1176 DavixError::clearError(&tmp_err);
1177 Uri new_url = Azure::transformURI(uri, p, true);
1178 DirHandle handle(new GetRequest(context, new_url, &tmp_err), new AzurePropParser(Azure::extract_azure_filename(uri)));
1179
1180 dav_ssize_t s_resu=0;
1181
1182 const int operation_timeout = p.getOperationTimeout()->tv_sec;
1183 HttpRequest & http_req = *(handle.request);
1184 XMLPropParser & parser = *(handle.parser);
1185
1186 time_t timestamp_timeout = time(NULL) + ((operation_timeout)?(operation_timeout):180);
1187
1188 http_req.setParameters(p);
1189
1190 http_req.beginRequest(&tmp_err);
1191 checkDavixError(&tmp_err);
1192 check_file_status(http_req, scope);
1193
1194 size_t prop_size = 0;
1195 do{ // first entry -> container information
1196 s_resu = incremental_listdir_parsing(&http_req, &parser, 2048, davix_scope_directory_listing_str());
1197
1198 prop_size = parser.getProperties().size();
1199 if(s_resu < 2048 && prop_size <1){ // verify request status : if req done + no data -> error
1200 throw DavixException(davix_scope_directory_listing_str(), StatusCode::IsNotADirectory, "The specified directory does not exist");
1201 }
1202 if(timestamp_timeout < time(NULL)){
1203 throw DavixException(davix_scope_directory_listing_str(), StatusCode::OperationTimeout, "Operation timeout triggered while directory listing");
1204 }
1205 }while( prop_size < 1); // prop < 1 means not enough data
1206
1207 st_info.mode = 0755;
1208 st_info.mode |= S_IFDIR;
1209 }
1210 // file exists, parse its info
1211 else if(code == 200) {
1212 st_info.mode = 0755;
1213 st_info.mode |= S_IFREG;
1214 const dav_ssize_t s = req.getAnswerSize();
1215 st_info.size = std::max<dav_ssize_t>(0,s);
1216 st_info.mtime = req.getLastModified();
1217 }
1218
1219 }
1220
1221 }
1222
1223 StatInfo & AzureMetaOps::statInfo(IOChainContext & iocontext, StatInfo & st_info) {
1224 if(is_azure_operation(iocontext)) {
1225 azureStatMapper(iocontext._context, iocontext._reqparams, iocontext._uri, st_info);
1226 return st_info;
1227 }
1228 else {
1229 return HttpIOChain::statInfo(iocontext, st_info);
1230 }
1231 }
1232
1233 static void azure_start_listing_query(std::unique_ptr<DirHandle> & handle, Context & context, const RequestParams* params, const Uri & url, const std::string & body) {
1234 DavixError* tmp_err=NULL;
1235 dav_ssize_t s_resu;
1236
1237 Uri new_url = Davix::Azure::transformURI(url, params, true);
1238 handle.reset(new DirHandle(new GetRequest(context, new_url, &tmp_err), new AzurePropParser(Davix::Azure::extract_azure_filename(url))));
1239
1240 const int operation_timeout = params->getOperationTimeout()->tv_sec;
1241 HttpRequest & http_req = *(handle->request);
1242 XMLPropParser & parser = *(handle->parser);
1243 time_t timestamp_timeout = time(NULL) + ((operation_timeout)?(operation_timeout):180);
1244
1245 http_req.setParameters(params);
1246
1247 http_req.beginRequest(&tmp_err);
1248 checkDavixError(&tmp_err);
1249
1250 check_file_status(http_req, davix_scope_directory_listing_str());
1251
1252 size_t prop_size = 0;
1253 do{ // first entry -> container information
1254 s_resu = incremental_listdir_parsing(&http_req, &parser, 2048, davix_scope_directory_listing_str());
1255
1256 prop_size = parser.getProperties().size();
1257 if(s_resu < 2048 && prop_size <1){ // verify request status : if req done + no data -> error
1258 throw DavixException(davix_scope_directory_listing_str(), StatusCode::IsNotADirectory, "The specified directory does not exist");
1259 }
1260 if(timestamp_timeout < time(NULL)){
1261 throw DavixException(davix_scope_directory_listing_str(), StatusCode::OperationTimeout, "Operation timeout triggered while directory listing");
1262 }
1263 }while( prop_size < 1); // prop < 1 means not enough data
1264
1265 }
1266
1267 bool azure_get_next_property(std::unique_ptr<DirHandle> & handle, std::string & name_entry, StatInfo & info) {
1268 DAVIX_SLOG(DAVIX_LOG_DEBUG, DAVIX_LOG_CHAIN, " -> azure_get_next_property");
1269 const size_t read_size = 2048;
1270
1271 HttpRequest& req = *(handle->request); // setup env again
1272 XMLPropParser& parser = *(handle->parser);
1273
1274 size_t prop_size = parser.getProperties().size();
1275 ssize_t s_resu = read_size;
1276
1277 while( prop_size == 0
1278 && s_resu > 0){ // execute request only if no property are available
1279
1280 // continue the parsing until one more result
1281 s_resu = incremental_listdir_parsing(&req, &parser, read_size, "S3::listing");
1282 prop_size = parser.getProperties().size();
1283 }
1284
1285 if(prop_size == 0){
1286 return false; // end of the request, end of the story
1287 }
1288
1289 FileProperties & front = parser.getProperties().front();
1290 name_entry.swap(front.filename);
1291 info = front.info;
1292 parser.getProperties().pop_front(); // clean the current element
1293 return true;
1294 }
1295
1296 static bool azure_directory_listing(std::unique_ptr<DirHandle> & handle, Context & context, const RequestParams* params, const Uri & uri, const std::string & body, std::string & name_entry, StatInfo & info){
1297 if(handle.get() == NULL){
1298 azure_start_listing_query(handle, context, params, uri, body);
1299 }
1300 return azure_get_next_property(handle, name_entry, info);
1301 }
1302
1303 bool AzureMetaOps::nextSubItem(IOChainContext &iocontext, std::string &entry_name, StatInfo &info) {
1304 if(is_azure_operation(iocontext)){
1305 return azure_directory_listing(directoryItem, iocontext._context, iocontext._reqparams, iocontext._uri, stat_listing,
1306 entry_name, info);
1307 }else{
1308 return HttpIOChain::nextSubItem(iocontext, entry_name, info);
1309 }
1310 }
1311
1312
1313 } // Davix