/* * This File is part of Davix, The IO library for HTTP based protocols * Copyright (C) CERN 2013 * Author: Adrien Devresse * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA * */ #include #include "davmeta.hpp" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "libs/alibxx/crypto/base64.hpp" #include using namespace StrUtil; namespace Davix{ static const std::string stat_listing("" "" "" "" "" ""); static const std::string quota_stat("" "" "" ""); struct DirHandle{ DirHandle(HttpRequest* req, XMLPropParser * p): request(req), parser(p){} std::unique_ptr request; std::unique_ptr parser; }; /** * Helper function to execute a PROPFIND/stat request on a given HTTP request handle. * Returns a string with the HTTP response. */ std::string req_webdav_propfind(HttpRequest* req, DavixError** err) { req->addHeaderField("Depth", "0"); req->setRequestMethod("PROPFIND"); if (req->executeRequest(err) == 0) { auto content = req->getAnswerContent(); if (content != NULL) { return {content}; } } return {}; } int dav_stat_mapper_webdav(Context &context, const RequestParams* params, const Uri & url, struct StatInfo& st_info){ DavPropXMLParser parser; DavixError* tmp_err = NULL; HttpRequest req(context, url, &tmp_err); int ret = -1; if( tmp_err == NULL){ req.setParameters(params); TRY_DAVIX { std::string response = req_webdav_propfind(&req, &tmp_err); if (!tmp_err) { parser.parseChunk(response); std::deque& props = parser.getProperties(); if (props.size() < 1) { throw DavixException(davix_scope_stat_str(), Davix::StatusCode::WebDavPropertiesParsingError, "Parsing Error: properties number < 1"); } else { st_info = props.front().info; ret = 0; } } } CATCH_DAVIX(&tmp_err) if (tmp_err != NULL) { ret = -1; } } checkDavixError(&tmp_err); return ret; } int dav_stat_mapper_http(Context& context, const RequestParams* params, const Uri & uri, struct StatInfo& st_info){ int ret = -1; DavixError * tmp_err=NULL; HeadRequest req(context, uri, &tmp_err); if( tmp_err == NULL){ req.setParameters(params); req.executeRequest(&tmp_err); if(!tmp_err){ if(httpcodeIsValid(req.getRequestCode()) ){ memset(&st_info, 0, sizeof(struct StatInfo)); const dav_ssize_t s = req.getAnswerSize(); st_info.size = std::max(0,s); st_info.mode = 0755 | S_IFREG; ret = 0; }else{ httpcodeToDavixError(req.getRequestCode(), davix_scope_http_request(), uri.getString() , &tmp_err); ret = -1; } } } checkDavixError(&tmp_err); return ret; } // Implement stat with a GET of Range 1 int dav_stat_mapper_http_get(Context& context, const RequestParams* params, const Uri & uri, struct StatInfo& st_info){ int ret = -1; DavixError * tmp_err=NULL; GetRequest req(context, uri, &tmp_err); if( tmp_err == NULL){ req.setParameters(params); req.addHeaderField("Range", "bytes=0-1"); req.executeRequest(&tmp_err); if(!tmp_err){ if(httpcodeIsValid(req.getRequestCode()) ){ memset(&st_info, 0, sizeof(struct StatInfo)); std::string rnge; req.getAnswerHeader("Content-Range", rnge); std::string::size_type pos = rnge.find_first_of("/"); if (pos == std::string::npos) { throw DavixException(davix_scope_meta(), StatusCode::ParsingError, "Content-Range not parsable"); } if (rnge.substr(pos+1,1) == "*") { throw DavixException(davix_scope_meta(), StatusCode::ParsingError, "Server does not provide content length"); } long lsize = toType()(rnge.substr(pos+1)); st_info.size = std::max(0,lsize); st_info.mode = 0755 | S_IFREG; req.discardBody(&tmp_err); ret = 0; }else{ httpcodeToDavixError(req.getRequestCode(), davix_scope_http_request(), uri.getString() , &tmp_err); ret = -1; } } } checkDavixError(&tmp_err); return ret; } dav_ssize_t incremental_listdir_parsing(HttpRequest* req, XMLPropParser * parser, dav_size_t s_buff, const std::string & scope){ DavixError* tmp_err=NULL; char buffer[s_buff+1]; const dav_ssize_t ret = req->readSegment(buffer, s_buff, &tmp_err); checkDavixError(&tmp_err); if(ret >= 0){ buffer[ret]= '\0'; parser->parseChunk(buffer, ret); }else{ throw DavixException(scope, StatusCode::UnknownError, "Unknown readSegment error"); } return ret; } dav_ssize_t getStatInfo(Context & c, const Uri & url, const RequestParams * p, struct StatInfo& st_info){ RequestParams params(p); configureRequestParamsProto(url, params); int ret =-1; switch(params.getProtocol()){ case RequestProtocol::Webdav: ret = dav_stat_mapper_webdav(c, ¶ms, url, st_info); break; default: if (isS3SignedURL(url)) { // This endpoint won't accept a HEAD request, use GET instead ret = dav_stat_mapper_http_get(c, ¶ms, url, st_info); } else { ret = dav_stat_mapper_http(c, ¶ms, url, st_info); } break; } DAVIX_SLOG(DAVIX_LOG_DEBUG, DAVIX_LOG_CHAIN, " davix_stat <-"); return ret; } QuotaInfo::QuotaInfo() : d_ptr(new Internal()) { } QuotaInfo::~QuotaInfo() { } dav_size_t QuotaInfo::getUsedBytes() { return d_ptr->used_bytes; } dav_size_t QuotaInfo::getFreeSpace() { return d_ptr->free_space; } class QuotaInfoHandler { public: static void setdptr(QuotaInfo &info, QuotaInfo::Internal &internal) { info.d_ptr.reset(new QuotaInfo::Internal(internal)); } }; void getQuotaInfo(Context & c, const Uri & url, const RequestParams *p, QuotaInfo &info) { DavixError * tmp_err = NULL; HttpRequest req(c, url, &tmp_err); checkDavixError(&tmp_err); req.setParameters(*p); req.addHeaderField("Depth","0"); req.setRequestMethod("PROPFIND"); req.setRequestBody(quota_stat); if (req.executeRequest(&tmp_err) == 0 && !tmp_err) { DavPropXMLParser parser; auto content = req.getAnswerContent(); std::string response = (content != NULL) ? content : ""; parser.parseChunk(response); std::deque & props = parser.getProperties(); if (props.size() < 1) { throw DavixException(davix_scope_stat_str(), Davix::StatusCode::WebDavPropertiesParsingError, "Parsing Error: properties number < 1"); } else { QuotaInfoHandler::setdptr(info, props.front().quota); } } checkDavixError(&tmp_err); } void parse_creation_deletion_result(int code, const Uri & u, const std::string & scope, const std::vector & body){ switch(code) { case 200: case 201: case 202: case 204: { return; } case 207: { // Parse WebDAV DavDeleteXMLParser parser; std::string response = (!body.empty()) ? body.data() : ""; parser.parseChunk(response); if (parser.getProperties().size() > 0) { for (unsigned int i = 0; i < parser.getProperties().size(); ++i) { const int sub_code = parser.getProperties().at(i).req_status; std::ostringstream ss; ss << "occurred during deletion request for " << parser.getProperties().at(i).filename; if (httpcodeIsValid(sub_code) == false) { httpcodeToDavixException(sub_code, scope, ss.str()); } } return; } // if no properties, properties were filtered because invalid httpcodeToDavixException(404, scope); break; } } std::ostringstream ss; ss << " with url " << u.getString(); httpcodeToDavixException(code, scope, ss.str()); } int internal_delete_resource(Context & c, const Uri & url, const RequestParams & params){ DavixError* tmp_err=NULL; int ret=-1; RequestParams _params(params); DeleteRequest req(c, url, &tmp_err); req.setParameters(_params); if(!tmp_err){ if( ( ret=req.executeRequest(&tmp_err)) == 0){ parse_creation_deletion_result(req.getRequestCode(), url, davix_scope_rm_str(), req.getAnswerContentVec()); } } checkDavixError(&tmp_err); return ret; } int internal_make_collection(Context & c, const Uri & url, const RequestParams & params){ DAVIX_SCOPE_TRACE(DAVIX_LOG_CHAIN, mkcoll); int ret=-1; DavixError* tmp_err=NULL; RequestParams _params(params); HttpRequest req(c, url, &tmp_err); if(tmp_err == NULL){ req.setParameters(params); req.setRequestMethod("MKCOL"); if( (ret = req.executeRequest(&tmp_err)) == 0){ parse_creation_deletion_result(req.getRequestCode(), url, davix_scope_mkdir_str(), req.getAnswerContentVec()); } } checkDavixError(&tmp_err); return ret; } int internal_move(Context & c, const Uri & url, const RequestParams & params, const std::string & target_url){ DAVIX_SLOG(DAVIX_LOG_DEBUG, DAVIX_LOG_CHAIN, " -> move"); int ret=-1; DavixError* tmp_err=NULL; RequestParams _params(params); HttpRequest req(c, url, &tmp_err); if(tmp_err == NULL){ req.setParameters(params); req.setRequestMethod("MOVE"); Uri uri(target_url); uri.httpizeProtocol(); req.addHeaderField("Destination", uri.getString()); if( (ret = req.executeRequest(&tmp_err)) == 0){ parse_creation_deletion_result(req.getRequestCode(), url, davix_scope_mv_str(), req.getAnswerContentVec()); } } DAVIX_SLOG(DAVIX_LOG_DEBUG, DAVIX_LOG_CHAIN, " move <-"); checkDavixError(&tmp_err); return ret; } int internal_checksum(Context & c, const Uri & url, const RequestParams *p, std::string & checksm, const std::string & chk_algo){ DAVIX_SLOG(DAVIX_LOG_DEBUG, DAVIX_LOG_CHAIN, " -> checksum"); int ret=-1; DavixError* tmp_err=NULL; RequestParams params(p); HeadRequest req(c, url, &tmp_err); if(tmp_err == NULL){ // add Digest file, support for other digest, extended format req.addHeaderField("Want-Digest", chk_algo); req.setParameters(params); if( (ret = req.executeRequest(&tmp_err)) == 0 && !tmp_err && (ret = davixRequestToFileStatus(&req, davix_scope_mkdir_str(), &tmp_err)) >=0){ // try simple MD5 ( standard ) if(compare_ncase(chk_algo, "MD5") == 0){ std::string chk; if(req.getAnswerHeader("Content-MD5", chk) == true){ DAVIX_SLOG(DAVIX_LOG_TRACE, DAVIX_LOG_CHAIN, "Extract MD5 checksum in base64 {}", chk); chk= Base64::base64_decode(chk); std::swap(checksm, chk); return 0; } } // fallback on extension for checksum HeaderVec headers; req.getAnswerHeaders(headers); if(ChecksumExtractor::extractChecksum(headers, chk_algo, checksm)) { return 0; } // last chance try to extract MD5 checksum from ETAG ( S3 work around ) std::string etag_str; if(compare_ncase(chk_algo, "MD5") ==0 && req.getAnswerHeader("etag", etag_str)){ stringVec tokens = tokenSplit(etag_str, "&;\\/\"'"); for(stringVec::iterator it = tokens.begin(); it < tokens.end(); it++){ if(it->size() == 32 && std::find_if(it->begin(), it->end(), std::not1(StrUtil::isHexa())) == it->end()){ std::swap(checksm, *it); return 0; } } } std::ostringstream ss; ss << "checksum calculation for " << chk_algo << " not supported for " << url; throw DavixException(davix_scope_meta(), StatusCode::OperationNonSupported, ss.str()); DAVIX_SLOG(DAVIX_LOG_DEBUG, DAVIX_LOG_CHAIN, " checksum <-"); return 0; } } throw DavixException(&tmp_err); } bool wedav_get_next_property(std::unique_ptr & handle, std::string & name_entry, StatInfo & info){ DAVIX_SLOG(DAVIX_LOG_DEBUG, DAVIX_LOG_CHAIN, " -> wedav_get_next_property"); const size_t read_size = 2048; HttpRequest& req = *(handle->request); // setup env again XMLPropParser& parser = *(handle->parser); size_t prop_size = parser.getProperties().size(); ssize_t s_resu = read_size; while( prop_size == 0 && s_resu > 0){ // request not complete and current data too smalls // continue the parsing until one more result s_resu = incremental_listdir_parsing(&req, &parser, read_size, "WebDav::listing"); prop_size = parser.getProperties().size(); } if(prop_size == 0){ return false; // end of the request, end of the story } FileProperties & front = parser.getProperties().front(); name_entry.swap(front.filename); info = front.info; parser.getProperties().pop_front(); // clean the current element return true; } void webdav_start_listing_query(std::unique_ptr & handle, Context & context, const RequestParams* params, const Uri & url, const std::string & body){ dav_ssize_t s_resu; DavixError* tmp_err=NULL; handle.reset(new DirHandle(new PropfindRequest(context, url, &tmp_err), new DavPropXMLParser())); checkDavixError(&tmp_err); HttpRequest & http_req = *(handle->request); XMLPropParser & parser = *(handle->parser); http_req.addHeaderField("Depth","1"); http_req.setParameters(params); // setup the handle for simple listing only http_req.setRequestBody(body); http_req.beginRequest(&tmp_err); checkDavixError(&tmp_err); check_file_status(http_req, davix_scope_directory_listing_str()); size_t prop_size = 0; do{ // parse the begining of the request until the first property -> directory property s_resu = incremental_listdir_parsing(&http_req, &parser, 2048, davix_scope_directory_listing_str()); prop_size = parser.getProperties().size(); if(s_resu < 2048 && prop_size <1){ // verify request status : if req done + no data -> error throw DavixException(davix_scope_directory_listing_str(), StatusCode::WebDavPropertiesParsingError, "bad server answer, not a valid WebDav PROPFIND answer"); } }while( prop_size < 1); // leave is end of req & no data const StatInfo & info = parser.getProperties().at(0).info; if( S_ISDIR(info.mode) == false){ std::ostringstream ss; ss << url << " is not a collection, listing impossible"; throw DavixException(davix_scope_directory_listing_str(), StatusCode::IsNotADirectory, ss.str()); }else{ parser.getProperties().pop_front(); // suppress the parent directory infos... } } bool webdav_directory_listing(std::unique_ptr & handle, Context & context, const RequestParams* params, const Uri & uri, const std::string & body, std::string & name_entry, StatInfo & info){ if(handle.get() == NULL){ webdav_start_listing_query(handle, context, params, uri, body); } return wedav_get_next_property(handle, name_entry, info); } HttpMetaOps::HttpMetaOps(): HttpIOChain(){} HttpMetaOps::~HttpMetaOps(){} void HttpMetaOps::checksum(IOChainContext & iocontext, std::string &checksm, const std::string &chk_algo){ internal_checksum(iocontext._context, iocontext._uri, iocontext._reqparams, checksm, chk_algo); } void HttpMetaOps::makeCollection(IOChainContext & iocontext){ internal_make_collection(iocontext._context, iocontext._uri, iocontext._reqparams); } void HttpMetaOps::move(IOChainContext & iocontext, const std::string & target_url){ internal_move(iocontext._context, iocontext._uri, iocontext._reqparams, target_url); } void HttpMetaOps::deleteResource(IOChainContext & iocontext){ internal_delete_resource(iocontext._context, iocontext._uri, iocontext._reqparams); } StatInfo & HttpMetaOps::statInfo(IOChainContext & iocontext, StatInfo &st_info){ struct stat st; memset(&st, 0, sizeof(struct stat)); getStatInfo(iocontext._context, iocontext._uri, iocontext._reqparams, st_info); return st_info; } QuotaInfo & HttpMetaOps::quotaInfo(IOChainContext & iocontext, QuotaInfo &info) { getQuotaInfo(iocontext._context, iocontext._uri, iocontext._reqparams, info); return info; } bool HttpMetaOps::nextSubItem(IOChainContext &iocontext, std::string &entry_name, StatInfo &info){ return webdav_directory_listing(directoryItem, iocontext._context, iocontext._reqparams, iocontext._uri, stat_listing, entry_name, info); } ///////////////////////// ///////////////////////// SwiftMetaOps::SwiftMetaOps() : HttpIOChain() {} SwiftMetaOps::~SwiftMetaOps(){} static bool is_swift_operation(IOChainContext & context){ return context._reqparams->getProtocol() == RequestProtocol::Swift; } static void swiftStatMapper(Context& context, const RequestParams* params, const Uri & uri, struct StatInfo& st_info) { const std::string scope = "Davix::swiftStatMapper"; DavixError * tmp_err=NULL; HeadRequest req(context, uri, &tmp_err); // we need to modify it, hence copy RequestParams p(params); if(tmp_err == NULL) { req.setParameters(p); req.executeRequest(&tmp_err); const int code = req.getRequestCode(); if(code == 404) { DavixError::clearError(&tmp_err); // try to "list" target resource and see if there is anything inside it, if there is, then it's a directory Uri new_url = Swift::swiftUriTransformer(uri, p, true); GetRequest http_req(context, new_url, &tmp_err); http_req.setParameters(p); http_req.beginRequest(&tmp_err); checkDavixError(&tmp_err); check_file_status(http_req, scope); // check response text, if there is data, then it is a directory, otherwise not a directory or file char buffer[256+1]; const dav_ssize_t ret = http_req.readSegment(buffer, 256, &tmp_err); checkDavixError(&tmp_err); if(ret == 0){ throw DavixException(scope, StatusCode::FileNotFound, "Not a file or directory"); } else if (ret < 0) { throw DavixException(scope, StatusCode::UnknownError, "Unknown readSegment error"); } checkDavixError(&tmp_err); st_info.mode = 0755; st_info.mode |= S_IFDIR; } else if(code == 200){ st_info.mode = 0755; std::string swift_path = Swift::extract_swift_path(uri); if(swift_path == "/") // is container st_info.mode |= S_IFDIR; else if(swift_path[swift_path.size()-1] == '/' && req.getAnswerSize() == 0) { // is a directory st_info.mode |= S_IFDIR; } else { // is file st_info.mode |= S_IFREG; const dav_ssize_t s = req.getAnswerSize(); st_info.size = std::max(0, s); st_info.mtime = req.getLastModified(); } } else if(code == 204){ // Normal response of a HEAD request to a container is 204 st_info.mode = 0755; std::string swift_path = Swift::extract_swift_path(uri); if(swift_path == "/") // is container st_info.mode |= S_IFDIR; } else if(code == 500){ throw DavixException(scope, StatusCode::UnknownError, "Internal Server Error triggered while attempting to get Swift object's stats"); } } checkDavixError(&tmp_err); } StatInfo & SwiftMetaOps::statInfo(IOChainContext &iocontext, StatInfo &st_info) { if(is_swift_operation(iocontext)){ swiftStatMapper(iocontext._context, iocontext._reqparams, iocontext._uri, st_info); return st_info; } else{ StatInfo & ref = HttpIOChain::statInfo(iocontext, st_info); return ref; } } bool is_a_container(Uri u) { std::string tmp = Swift::extract_swift_path(u); if(tmp.compare("/") == 0){ return true; } return false; } bool s3_get_next_property(std::unique_ptr & handle, std::string & name_entry, StatInfo & info); void swift_start_listing_query(std::unique_ptr & handle, Context & context, const RequestParams* params, const Uri & url, const std::string & body){ (void) body; dav_ssize_t s_resu; DavixError* tmp_err=NULL; if(params->getSwiftListingMode() == SwiftListingMode::Hierarchical){ Uri new_url = Swift::swiftUriTransformer(url, params, true); handle.reset(new DirHandle(new GetRequest(context, new_url, &tmp_err), new SwiftPropParser(Swift::extract_swift_path(url)))); } else if(params->getSwiftListingMode() == SwiftListingMode::SemiHierarchical){ Uri new_url = Swift::swiftUriTransformer(url, params, false); handle.reset(new DirHandle(new GetRequest(context, new_url, &tmp_err), new SwiftPropParser(Swift::extract_swift_path(url)))); } else{ if(is_a_container(url) == false){ throw DavixException(davix_scope_directory_listing_str(), StatusCode::IsNotADirectory, "This is not a Swift container"); } handle.reset(new DirHandle(new GetRequest(context, url, &tmp_err), new SwiftPropParser())); } checkDavixError(&tmp_err); const int operation_timeout = params->getOperationTimeout()->tv_sec; HttpRequest & http_req = *(handle->request); XMLPropParser & parser = *(handle->parser); time_t timestamp_timeout = time(NULL) + ((operation_timeout)?(operation_timeout):180); http_req.setParameters(params); http_req.addHeaderField("Accept", "application/xml"); http_req.beginRequest(&tmp_err); checkDavixError(&tmp_err); check_file_status(http_req, davix_scope_directory_listing_str()); size_t prop_size = 0; do{ // first entry -> container information s_resu = incremental_listdir_parsing(&http_req, &parser, 2048, davix_scope_directory_listing_str()); prop_size = parser.getProperties().size(); if(s_resu < 2048 && prop_size <1){ // verify request status : if req done + no data -> error throw DavixException(davix_scope_directory_listing_str(), StatusCode::ParsingError, "Invalid server response, not a Swift listing or the directory is empty"); } if(timestamp_timeout < time(NULL)){ throw DavixException(davix_scope_directory_listing_str(), StatusCode::OperationTimeout, "Operation timeout triggered while directory listing"); } }while( prop_size < 1); // prop < 1 means not enough data } bool swift_directory_listing(std::unique_ptr & handle, Context & context, const RequestParams* params, const Uri & uri, const std::string & body, std::string & name_entry, StatInfo & info){ if(handle.get() == NULL){ swift_start_listing_query(handle, context, params, uri, body); } return s3_get_next_property(handle, name_entry, info); } bool SwiftMetaOps::nextSubItem(IOChainContext &iocontext, std::string &entry_name, StatInfo &info){ if(is_swift_operation(iocontext)){ return swift_directory_listing(directoryItem, iocontext._context, iocontext._reqparams, iocontext._uri, stat_listing, entry_name, info); }else{ return HttpIOChain::nextSubItem(iocontext, entry_name, info); } } void SwiftMetaOps::move(IOChainContext & iocontext, const std::string & target_url) { const std::string scope = "Davix::SwiftMetaOps::move"; if(!is_swift_operation(iocontext)) { return HttpIOChain::move(iocontext, target_url); } Context context = iocontext._context; RequestParams params = iocontext._reqparams; Uri uri(iocontext._uri); Uri target(target_url); // verify both are using the same swift provider/server, reuse s3 utils std::string p1 = S3::extract_s3_provider(uri); std::string p2 = S3::extract_s3_provider(target); if(p1 != p2) { throw DavixException(scope, StatusCode::OperationNonSupported, "It looks that the two URLs are not using the same Swift provider. Unable to perform the move operation."); } std::string source_container = Swift::extract_swift_container(uri); std::string source_path = Swift::extract_swift_path(uri); DavixError *tmp_err = NULL; PutRequest req(context, target, &tmp_err); checkDavixError(&tmp_err); req.setParameters(iocontext._reqparams); req.addHeaderField("X-Copy-From", "/" + source_container + source_path); req.executeRequest(&tmp_err); checkDavixError(&tmp_err); // if copying was successful, delete the source file if(req.getRequestCode() == 201) { DeleteRequest req(context, uri, &tmp_err); checkDavixError(&tmp_err); RequestParams p(iocontext._reqparams); req.setParameters(p); req.executeRequest(&tmp_err); checkDavixError(&tmp_err); } else { std::stringstream str; str << "Received code " << req.getRequestCode() << " when trying to copy file - will not perform deletion"; throw DavixException(scope, StatusCode::UnknownError, str.str()); } } ///////////////////////// ///////////////////////// bool is_a_bucket(const Uri & u){ const std::string & s = u.getPath(); return (std::find_if(s.begin(), s.end(), std::not1(StrUtil::isSlash())) == s.end()); // false if pathname does not match '\/+' } S3MetaOps::S3MetaOps() : HttpIOChain() {} S3MetaOps::~S3MetaOps(){} static bool is_s3_operation(IOChainContext & context){ const std::string & proto = context._uri.getProtocol(); const RequestProtocol::Protocol protocol_flag = context._reqparams->getProtocol(); if( proto.compare(0, 2, "s3") ==0 || protocol_flag == RequestProtocol::AwsS3) return true; if( proto.compare(0, 6, "gcloud") ==0 || protocol_flag == RequestProtocol::Gcloud) return true; return false; } static void internal_s3_create_bucket_or_dir(Context & c, const Uri & url, const RequestParams & params){ DavixError * tmp_err=NULL; /* make sure path ends with a slash, otherwise s3 will just create a zero-length file */ Uri url2 = url; if(url.getPath()[url.getPath().size()-1] != '/') { url2.setPath(url.getPath() + "/"); } PutRequest req(c, url2, &tmp_err); req.addHeaderField("Content-Length", "0"); checkDavixError(&tmp_err); req.setParameters(params); if( req.executeRequest(&tmp_err) < 0){ const int code = req.getRequestCode(); httpcodeToDavixException(code, davix_scope_meta(), "bucket creation failure"); } checkDavixError(&tmp_err); } void S3MetaOps::move(IOChainContext & iocontext, const std::string & target_url) { const std::string scope = "Davix::S3MetaOps::move"; if(!is_s3_operation(iocontext)) { return HttpIOChain::move(iocontext, target_url); } Context context = iocontext._context; RequestParams params = iocontext._reqparams; Uri uri(iocontext._uri); Uri target(target_url); // verify both are using the same s3 provider/server std::string p1 = S3::extract_s3_provider(uri); std::string p2 = S3::extract_s3_provider(target); if(p1 != p2) { throw DavixException(scope, StatusCode::OperationNonSupported, "It looks that the two URLs are not using the same S3 provider. Unable to perform the move operation."); } std::string source_bucket = S3::extract_s3_bucket(uri, params.getAwsAlternate()); std::string source_path = S3::extract_s3_path(uri, params.getAwsAlternate()); DavixError *tmp_err = NULL; PutRequest req(context, target, &tmp_err); checkDavixError(&tmp_err); req.setParameters(iocontext._reqparams); req.addHeaderField("x-amz-copy-source", "/" + source_bucket + source_path); req.executeRequest(&tmp_err); checkDavixError(&tmp_err); // if copying was successful, delete the source file if(req.getRequestCode() == 200) { std::string region = S3::detect_region(uri); DAVIX_SLOG(DAVIX_LOG_DEBUG, DAVIX_LOG_CHAIN, "Detected region for source endpoint: " + region); checkDavixError(&tmp_err); DeleteRequest req(context, uri, &tmp_err); checkDavixError(&tmp_err); RequestParams p(iocontext._reqparams); p.setAwsRegion(region); req.setParameters(p); req.executeRequest(&tmp_err); checkDavixError(&tmp_err); } else { std::stringstream str; str << "Received code " << req.getRequestCode() << " when trying to copy file - will not perform deletion"; throw DavixException(scope, StatusCode::UnknownError, str.str()); } } void S3MetaOps::checksum(IOChainContext &iocontext, std::string &checksm, const std::string &chk_algo){ internal_checksum(iocontext._context, iocontext._uri, iocontext._reqparams, checksm, chk_algo); } void S3MetaOps::makeCollection(IOChainContext &iocontext){ if(is_s3_operation(iocontext) || is_swift_operation(iocontext)){ internal_s3_create_bucket_or_dir( iocontext._context, iocontext._uri, iocontext._reqparams); }else{ HttpIOChain::makeCollection(iocontext); } } void s3StatMapper(Context& context, const RequestParams* params, const Uri & uri, struct StatInfo& st_info){ const std::string scope = "Davix::s3StatMapper"; DavixError * tmp_err=NULL; HeadRequest req(context, uri, &tmp_err); // we need to modify it, hence copy RequestParams p(params); // we just need to know if target has anything inside it p.setS3MaxKey(1); if( tmp_err == NULL){ req.setParameters(p); req.executeRequest(&tmp_err); const int code = req.getRequestCode(); // if 404, target either doesn't exist or is a S3 "directory" if(code == 404){ DavixError::clearError(&tmp_err); // try to "list" target resource and see if there is anything inside it, if there is, then it's a directory Uri new_url = S3::s3UriTransformer(uri, p, true); DirHandle handle(new GetRequest(context, new_url, &tmp_err), new S3PropParser(params->getS3ListingMode(), S3::extract_s3_path(uri, params->getAwsAlternate()))); dav_ssize_t s_resu=0; const int operation_timeout = p.getOperationTimeout()->tv_sec; HttpRequest & http_req = *(handle.request); XMLPropParser & parser = *(handle.parser); time_t timestamp_timeout = time(NULL) + ((operation_timeout)?(operation_timeout):180); http_req.setParameters(p); http_req.beginRequest(&tmp_err); checkDavixError(&tmp_err); check_file_status(http_req, scope); size_t prop_size = 0; do{ // first entry TRY_DAVIX{ s_resu = incremental_listdir_parsing(&http_req, &parser, 2048, scope); }CATCH_DAVIX(&tmp_err) if(tmp_err && (tmp_err->getStatus() == StatusCode::IsNotADirectory)){ std::ostringstream ss; ss << uri << " not found"; throw DavixException(scope, StatusCode::FileNotFound, ss.str()); } prop_size = parser.getProperties().size(); if(s_resu < 2048 && prop_size <1){ // verify request status : if req done + no data -> error throw DavixException(scope, StatusCode::ParsingError, "Invalid server response, not a S3 listing"); } if(timestamp_timeout < time(NULL)){ throw DavixException(scope, StatusCode::OperationTimeout, "Operation timeout triggered while getting S3 object's stats"); } }while( prop_size < 1); // prop < 1 means not enough data st_info.mode = 0755; st_info.mode |= S_IFDIR; } else if(code == 200){ // found something, must be a file not directory st_info.mode = 0755; std::string s3_path = S3::extract_s3_path(uri, params->getAwsAlternate()); if(s3_path == "/") // is bucket st_info.mode |= S_IFDIR; else if(s3_path[s3_path.size()-1] == '/' && req.getAnswerSize() == 0) { // is a directory st_info.mode |= S_IFDIR; } else{ // is file st_info.mode |= S_IFREG; const dav_ssize_t s = req.getAnswerSize(); st_info.size = std::max(0,s); st_info.mtime = req.getLastModified(); } } else if(code == 500) throw DavixException(scope, StatusCode::UnknownError, "Internal Server Error triggered while attempting to get S3 object's stats"); } checkDavixError(&tmp_err); } // get statInfo StatInfo & S3MetaOps::statInfo(IOChainContext & iocontext, StatInfo & st_info){ if(is_s3_operation(iocontext)){ s3StatMapper(iocontext._context, iocontext._reqparams, iocontext._uri, st_info); return st_info; } else{ StatInfo & ref = HttpIOChain::statInfo(iocontext, st_info); return ref; } } bool s3_get_next_property(std::unique_ptr & handle, std::string & name_entry, StatInfo & info){ DAVIX_SLOG(DAVIX_LOG_DEBUG, DAVIX_LOG_CHAIN, " -> s3_get_next_property"); const size_t read_size = 2048*1024; HttpRequest& req = *(handle->request); // setup env again XMLPropParser& parser = *(handle->parser); size_t prop_size = parser.getProperties().size(); ssize_t s_resu = read_size; while( prop_size == 0 && s_resu > 0){ // execute request only if no property are available // continue the parsing until one more result s_resu = incremental_listdir_parsing(&req, &parser, read_size, "S3::listing"); prop_size = parser.getProperties().size(); } if(prop_size == 0){ return false; // end of the request, end of the story } FileProperties & front = parser.getProperties().front(); name_entry.swap(front.filename); info = front.info; parser.getProperties().pop_front(); // clean the current element return true; } void s3_start_listing_query(std::unique_ptr & handle, Context & context, const RequestParams* params, const Uri & url, const std::string & body){ (void) body; dav_ssize_t s_resu; DavixError* tmp_err=NULL; bool listing_buckets; if(params->getProtocol() == RequestProtocol::Gcloud) { Uri new_url = gcloud::getListingURI(url, params); std::string prefix = gcloud::extract_path(url); if(prefix != "/") prefix = "/" + prefix; handle.reset(new DirHandle(new GetRequest(context, new_url, &tmp_err), new S3PropParser(params->getS3ListingMode(), prefix))); } else if(params->getS3ListingMode() == S3ListingMode::Hierarchical){ Uri new_url = S3::s3UriTransformer(url, params, true); if (handle.get() != NULL) { XMLPropParser & parser = *(handle->parser); std::string nextmarker = parser.getNextMarker(); if (nextmarker != "") { new_url.addQueryParam("marker", nextmarker); } } handle.reset(new DirHandle(new GetRequest(context, new_url, &tmp_err), new S3PropParser(params->getS3ListingMode(), S3::extract_s3_path(url, params->getAwsAlternate())))); } else if(params->getS3ListingMode() == S3ListingMode::SemiHierarchical){ Uri new_url = S3::s3UriTransformer(url, params, false); handle.reset(new DirHandle(new GetRequest(context, new_url, &tmp_err), new S3PropParser(params->getS3ListingMode(), S3::extract_s3_path(url, params->getAwsAlternate())))); } else{ if(is_a_bucket(url) == false){ throw DavixException(davix_scope_directory_listing_str(), StatusCode::IsNotADirectory, "This is not a S3 bucket"); } handle.reset(new DirHandle(new GetRequest(context, url, &tmp_err), new S3PropParser())); } checkDavixError(&tmp_err); // Check if we are listing available buckets listing_buckets = (params->getAwsAlternate() && url.getPath() == "/"); const int operation_timeout = params->getOperationTimeout()->tv_sec; HttpRequest & http_req = *(handle->request); XMLPropParser & parser = *(handle->parser); time_t timestamp_timeout = time(NULL) + ((operation_timeout)?(operation_timeout):180); http_req.setParameters(params); http_req.beginRequest(&tmp_err); checkDavixError(&tmp_err); check_file_status(http_req, davix_scope_directory_listing_str()); size_t prop_size = 0; do{ // first entry -> bucket information s_resu = incremental_listdir_parsing(&http_req, &parser, 2048*1024, davix_scope_directory_listing_str()); prop_size = parser.getProperties().size(); if(s_resu < 2048*1024 && prop_size <1){ // verify request status : if req done + no data -> error throw DavixException(davix_scope_directory_listing_str(), StatusCode::ParsingError, "Invalid server response, not a S3 listing"); } if(timestamp_timeout < time(NULL)){ throw DavixException(davix_scope_directory_listing_str(), StatusCode::OperationTimeout, "Operation timeout triggered while directory listing"); } }while( prop_size < 1); // prop < 1 means not enough data const StatInfo & info = parser.getProperties().at(0).info; if( S_ISDIR(info.mode) == false){ std::ostringstream ss; ss << url << " is not a S3 bucket"; throw DavixException(davix_scope_directory_listing_str(), StatusCode::IsNotADirectory, ss.str()); } else if (!listing_buckets) { parser.getProperties().pop_front(); // suppress the bucket name entry } } bool s3_directory_listing(std::unique_ptr & handle, Context & context, const RequestParams* params, const Uri & uri, const std::string & body, std::string & name_entry, StatInfo & info){ if (handle.get() == NULL){ s3_start_listing_query(handle, context, params, uri, body); } else { XMLPropParser& parser = *(handle->parser); if (parser.getProperties().empty() && !parser.getNextMarker().empty()) { s3_start_listing_query(handle, context, params, uri, body); } } return s3_get_next_property(handle, name_entry, info); } bool S3MetaOps::nextSubItem(IOChainContext &iocontext, std::string &entry_name, StatInfo &info){ if(is_s3_operation(iocontext)){ return s3_directory_listing(directoryItem, iocontext._context, iocontext._reqparams, iocontext._uri, stat_listing, entry_name, info); }else{ return HttpIOChain::nextSubItem(iocontext, entry_name, info); } } ///////////////////////// ///////////////////////// AzureMetaOps::AzureMetaOps() : HttpIOChain() {} AzureMetaOps::~AzureMetaOps(){} static bool is_azure_operation(IOChainContext & context){ return context._reqparams->getProtocol() == RequestProtocol::Azure; } void azureStatMapper(Context& context, const RequestParams* params, const Uri & uri, struct StatInfo& st_info) { const std::string scope = "Davix::azureStatMapper"; DavixError * tmp_err=NULL; //Context c; HeadRequest req(context, uri, &tmp_err); // we need to modify it, hence copy RequestParams p(params); // we just need to know if target has anything inside it //p.setS3MaxKey(1); if( tmp_err == NULL){ req.setParameters(p); req.executeRequest(&tmp_err); const int code = req.getRequestCode(); // if 404, target either doesn't exist or is an Azure "directory". TODO: add support to stat directories. must have func tests by then if(code == 404){ DavixError::clearError(&tmp_err); Uri new_url = Azure::transformURI(uri, p, true); DirHandle handle(new GetRequest(context, new_url, &tmp_err), new AzurePropParser(Azure::extract_azure_filename(uri))); dav_ssize_t s_resu=0; const int operation_timeout = p.getOperationTimeout()->tv_sec; HttpRequest & http_req = *(handle.request); XMLPropParser & parser = *(handle.parser); time_t timestamp_timeout = time(NULL) + ((operation_timeout)?(operation_timeout):180); http_req.setParameters(p); http_req.beginRequest(&tmp_err); checkDavixError(&tmp_err); check_file_status(http_req, scope); size_t prop_size = 0; do{ // first entry -> container information s_resu = incremental_listdir_parsing(&http_req, &parser, 2048, davix_scope_directory_listing_str()); prop_size = parser.getProperties().size(); if(s_resu < 2048 && prop_size <1){ // verify request status : if req done + no data -> error throw DavixException(davix_scope_directory_listing_str(), StatusCode::IsNotADirectory, "The specified directory does not exist"); } if(timestamp_timeout < time(NULL)){ throw DavixException(davix_scope_directory_listing_str(), StatusCode::OperationTimeout, "Operation timeout triggered while directory listing"); } }while( prop_size < 1); // prop < 1 means not enough data st_info.mode = 0755; st_info.mode |= S_IFDIR; } // file exists, parse its info else if(code == 200) { st_info.mode = 0755; st_info.mode |= S_IFREG; const dav_ssize_t s = req.getAnswerSize(); st_info.size = std::max(0,s); st_info.mtime = req.getLastModified(); } } } StatInfo & AzureMetaOps::statInfo(IOChainContext & iocontext, StatInfo & st_info) { if(is_azure_operation(iocontext)) { azureStatMapper(iocontext._context, iocontext._reqparams, iocontext._uri, st_info); return st_info; } else { return HttpIOChain::statInfo(iocontext, st_info); } } static void azure_start_listing_query(std::unique_ptr & handle, Context & context, const RequestParams* params, const Uri & url, const std::string & body) { DavixError* tmp_err=NULL; dav_ssize_t s_resu; Uri new_url = Davix::Azure::transformURI(url, params, true); handle.reset(new DirHandle(new GetRequest(context, new_url, &tmp_err), new AzurePropParser(Davix::Azure::extract_azure_filename(url)))); const int operation_timeout = params->getOperationTimeout()->tv_sec; HttpRequest & http_req = *(handle->request); XMLPropParser & parser = *(handle->parser); time_t timestamp_timeout = time(NULL) + ((operation_timeout)?(operation_timeout):180); http_req.setParameters(params); http_req.beginRequest(&tmp_err); checkDavixError(&tmp_err); check_file_status(http_req, davix_scope_directory_listing_str()); size_t prop_size = 0; do{ // first entry -> container information s_resu = incremental_listdir_parsing(&http_req, &parser, 2048, davix_scope_directory_listing_str()); prop_size = parser.getProperties().size(); if(s_resu < 2048 && prop_size <1){ // verify request status : if req done + no data -> error throw DavixException(davix_scope_directory_listing_str(), StatusCode::IsNotADirectory, "The specified directory does not exist"); } if(timestamp_timeout < time(NULL)){ throw DavixException(davix_scope_directory_listing_str(), StatusCode::OperationTimeout, "Operation timeout triggered while directory listing"); } }while( prop_size < 1); // prop < 1 means not enough data } bool azure_get_next_property(std::unique_ptr & handle, std::string & name_entry, StatInfo & info) { DAVIX_SLOG(DAVIX_LOG_DEBUG, DAVIX_LOG_CHAIN, " -> azure_get_next_property"); const size_t read_size = 2048; HttpRequest& req = *(handle->request); // setup env again XMLPropParser& parser = *(handle->parser); size_t prop_size = parser.getProperties().size(); ssize_t s_resu = read_size; while( prop_size == 0 && s_resu > 0){ // execute request only if no property are available // continue the parsing until one more result s_resu = incremental_listdir_parsing(&req, &parser, read_size, "S3::listing"); prop_size = parser.getProperties().size(); } if(prop_size == 0){ return false; // end of the request, end of the story } FileProperties & front = parser.getProperties().front(); name_entry.swap(front.filename); info = front.info; parser.getProperties().pop_front(); // clean the current element return true; } static bool azure_directory_listing(std::unique_ptr & handle, Context & context, const RequestParams* params, const Uri & uri, const std::string & body, std::string & name_entry, StatInfo & info){ if(handle.get() == NULL){ azure_start_listing_query(handle, context, params, uri, body); } return azure_get_next_property(handle, name_entry, info); } bool AzureMetaOps::nextSubItem(IOChainContext &iocontext, std::string &entry_name, StatInfo &info) { if(is_azure_operation(iocontext)){ return azure_directory_listing(directoryItem, iocontext._context, iocontext._reqparams, iocontext._uri, stat_listing, entry_name, info); }else{ return HttpIOChain::nextSubItem(iocontext, entry_name, info); } } } // Davix