Debian Codegraph @

src/tools/davix_op.cpp

raw
1 /*
2 * This File is part of Davix, The IO library for HTTP based protocols
3 * Copyright (C) CERN 2013
4 * Author: Kwong Tat Cheung <[email protected]>
5 *
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2.1 of the License, or (at your option) any later version.
10 *
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, write to the Free Software
18 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19 *
20 */
21
22 #include "davix_op.hpp"
23 #include "davix_taskqueue.hpp"
24 #include <utils/davix_logger_internal.hpp>
25 #include <sstream>
26 #include <xml/s3deleteparser.hpp>
27 #include <xml/davdeletexmlparser.hpp>
28
29
30 namespace Davix{
31
32 //-------------------------------------------------
33 //----------------------DavixOp--------------------
34 //-------------------------------------------------
35 DavixOp::DavixOp(const Tool::OptParams& opts, std::string target_url, std::string destination_url, Context& c) :
36 _target_url(target_url),
37 _destination_url(destination_url),
38 _opts(opts),
39 _c(c),
40 _scope()
41 {
42 }
43
44 DavixOp::~DavixOp(){}
45
46 std::string DavixOp::getTargetUrl(){
47 return _target_url;
48 }
49
50 std::string DavixOp::getDestinationUrl(){
51 return _destination_url;
52 }
53
54 std::string DavixOp::getOpType(){
55 return opType;
56 }
57
58
59 //-------------------------------------------------
60 //----------------------GetOp----------------------
61 //-------------------------------------------------
62 GetOp::GetOp(const Tool::OptParams& opts, std::string target_url, std::string destination_url, Context& c) :
63 DavixOp(opts, target_url, destination_url, c)
64 {
65 opType = "GET";
66 _scope = "Davix::DavixOp::GetOp";
67 }
68
69 GetOp::~GetOp(){}
70
71 int GetOp::executeOp(){
72 int ret = -1;
73 int fd = -1;
74 DavixError* tmp_err=NULL;
75 DavFile f(_c, _target_url);
76
77 if((fd = getOutFd())> 0){
78 DAVIX_SLOG(DAVIX_LOG_DEBUG, DAVIX_LOG_CORE, "{} executing op on ", _scope, _target_url);
79 ret = f.getToFd(&_opts.params, fd, &tmp_err);
80
81 //if getToFd failed, remove the just created blank local file
82 if(tmp_err){
83 std::cerr << std::endl << _scope << " Failed to GET " << _target_url << std::endl;
84 Tool::errorPrint(&tmp_err);
85 remove(_destination_url.c_str());
86 }
87 close(fd);
88 }
89 return ret;
90 }
91
92 int GetOp::getOutFd(){
93 DavixError* tmp_err=NULL;
94 int fd = -1;
95 if(_destination_url.empty() == false){
96 // for S3 we have to split the key into tokens and create a dir for each token
97 if (_opts.params.getProtocol() == RequestProtocol::AwsS3){
98 if((Tool::mkdirP(_destination_url, true)) < 0 ){
99 std::cerr << std::endl << _scope << " Failed to create local directory for " << _destination_url << std::endl;
100 return -1;
101 }
102 }
103 if((fd = open(_destination_url.c_str(), O_WRONLY | O_TRUNC | O_CREAT, 0777)) <0 ){
104 davix_errno_to_davix_error(errno, _scope, std::string("while opening/creating ").append(_destination_url), &tmp_err);
105 if(tmp_err){
106 Tool::errorPrint(&tmp_err);
107 }
108 return -1;
109 }
110
111 }else{
112 fd = dup(STDOUT_FILENO);
113 }
114 return fd;
115 }
116
117
118 //-------------------------------------------------
119 //----------------------PutOp----------------------
120 //-------------------------------------------------
121 PutOp::PutOp(const Tool::OptParams& opts, std::string target_url, std::string destination_url, dav_size_t file_size, Context& c) :
122 DavixOp(opts, target_url, destination_url, c)
123 {
124 opType = "PUT";
125 _scope = "Davix::DavixOp::PutOp";
126 _file_size = file_size;
127
128 }
129
130 PutOp::~PutOp(){}
131
132 int PutOp::executeOp(){
133 DavixError* tmp_err=NULL;
134 int fd = -1;
135
136 if( (fd = getInFd(&tmp_err)) < 0){
137 if(tmp_err)
138 Tool::errorPrint(&tmp_err);
139 return -1;
140 }
141
142 TRY_DAVIX{
143 DavFile f(_c, _destination_url);
144 DAVIX_SLOG(DAVIX_LOG_DEBUG, DAVIX_LOG_CORE, "{} executing op on ", _scope, _destination_url);
145 f.put(&_opts.params, fd, _file_size);
146 close(fd);
147 return 0;
148 }CATCH_DAVIX(&tmp_err);
149
150 if(fd != -1)
151 close(fd);
152
153 if(tmp_err->getStatus() == StatusCode::FileExist){
154 std::cerr << std::endl << _scope << " " << _destination_url << " already exists, continuing..." << std::endl;
155 }
156 else
157 std::cerr << std::endl << _scope << " Failed to PUT " << _target_url << std::endl;
158
159 Tool::errorPrint(&tmp_err);
160 return -1;
161 }
162
163 int PutOp::getInFd(DavixError** err){
164 int fd = -1;
165 if(_target_url.empty() == false){
166 if((fd = open(_target_url.c_str(), O_RDONLY)) <0 ){
167 davix_errno_to_davix_error(errno, _scope, std::string("for source file ").append(_target_url), err);
168 return -1;
169 }
170 }else{
171 fd = dup(STDOUT_FILENO);
172 }
173 return fd;
174
175 }
176
177
178 //-------------------------------------------------
179 //--------------------DeleteOp---------------------
180 //-------------------------------------------------
181 DeleteOp::DeleteOp(const Tool::OptParams& opts, std::string destination_url, Context& c, std::string buf) :
182 DavixOp(opts, "", destination_url, c)
183 {
184 opType = "DELETE";
185 _scope = "Davix::DavixOp::DeleteOp";
186 _buf = buf;
187 }
188
189 DeleteOp::~DeleteOp(){}
190
191 int DeleteOp::executeOp(){
192 DavixError* tmp_err=NULL;
193
194 DAVIX_SLOG(DAVIX_LOG_DEBUG, DAVIX_LOG_CORE, "{} executing op on ", _scope, _destination_url);
195
196 if(_opts.params.getProtocol() == RequestProtocol::AwsS3){
197 _destination_url += "/?delete";
198 PostRequest req(_c, _destination_url, &tmp_err);
199
200 if(tmp_err){
201 Tool::errorPrint(&tmp_err);
202 return -1;
203 }
204
205 req.setParameters(_opts.params);
206
207 std::ostringstream ss;
208 ss << _buf.size();
209
210 // calculate md5 of body and set header fields, these are required for S3 multi-objects delete
211 std::string md5;
212 S3::calculateMD5(_buf, md5);
213
214 req.addHeaderField("Content-MD5", md5);
215 req.addHeaderField("Content-Length", ss.str());
216
217 req.setRequestBody(_buf);
218
219 req.executeRequest(&tmp_err);
220
221 if(tmp_err){
222 Tool::errorPrint(&tmp_err);
223 return -1;
224 }
225
226 // check response code
227 int code = req.getRequestCode();
228
229 if(!httpcodeIsValid(code)){
230 httpcodeToDavixError(req.getRequestCode(), davix_scope_http_request(), "during S3 multi-objects delete operation", &tmp_err);
231 if(tmp_err){
232 Tool::errorPrint(&tmp_err);
233 return -1;
234 }
235 }
236
237 std::vector<char> body = req.getAnswerContentVec();
238
239 TRY_DAVIX{
240 parse_deletion_result(code, Uri(_destination_url), _scope, body);
241 }CATCH_DAVIX(&tmp_err);
242
243 if(tmp_err){
244 Tool::errorPrint(&tmp_err);
245 return -1;
246 }
247 }
248 else{
249 // Cases other than S3 not implemented for now. WebDAV delete collection already works without the -r switch
250 return -1;
251 }
252
253 return 0;
254 }
255
256 void DeleteOp::parse_deletion_result(int code, const Uri & u, const std::string & scope, const std::vector<char> & body){
257 switch(code) {
258 case 200: {
259 // if s3 && scope was davix_scope_rm_str() && batch delete, parse body
260 S3DeleteParser parser;
261 std::string response = (!body.empty()) ? body.data() : "";
262 parser.parseChunk(response);
263
264 // check if any of the delete request entry is flagged as error, if so, just print them for now
265 if (parser.getDeleteStatus().size() > 0) {
266 for (unsigned int i = 0; i < parser.getDeleteStatus().size(); ++i) {
267 if (parser.getDeleteStatus().at(i).error) {
268 std::ostringstream ss;
269 ss << "Error: " << parser.getDeleteStatus().at(i).error_code <<
270 " -> " << parser.getDeleteStatus().at(i).message <<
271 " encountered while attempting to delete " <<
272 parser.getDeleteStatus().at(i).filename;
273
274 std::cerr << std::endl << ss.str() << std::endl;
275 }
276
277 return;
278 }
279
280 // if no properties, status were filtered because invalid
281 httpcodeToDavixException(404, scope);
282 return;
283 }
284 }
285 case 201:
286 case 202:
287 case 204: {
288 return;
289 }
290 case 207: {
291 // Parse WebDAV
292 DavDeleteXMLParser parser;
293 std::string response = (!body.empty()) ? body.data() : "";
294 parser.parseChunk(response);
295
296 if (parser.getProperties().size() > 0) {
297 for (unsigned int i = 0; i < parser.getProperties().size(); ++i) {
298 const int sub_code = parser.getProperties().at(i).req_status;
299 std::ostringstream ss;
300 ss << "occurred during deletion request for " << parser.getProperties().at(i).filename;
301
302 if (httpcodeIsValid(sub_code) == false) {
303 httpcodeToDavixException(sub_code, scope, ss.str());
304 }
305 }
306
307 return;
308 }
309
310 // if no properties, properties were filtered because invalid
311 httpcodeToDavixException(404, scope);
312 break;
313 }
314 }
315
316 std::ostringstream ss;
317 ss << " with url " << u.getString();
318 httpcodeToDavixException(code, scope, ss.str());
319 }
320
321
322 //-------------------------------------------------
323 //---------------------ListOp----------------------
324 //-------------------------------------------------
325 ListOp::ListOp(const Tool::OptParams& opts, std::string target_url, Context& c, DavixTaskQueue* listing_tq, FILE* filestream, pthread_mutex_t& output_mutex) :
326 DavixOp(opts, target_url, "NULL", c),
327 _listing_tq(listing_tq),
328 _filestream(filestream),
329 _output_mutex(output_mutex)
330 {
331 opType = "LIST";
332 _scope = "Davix::DavixOp::ListOp";
333 }
334
335 ListOp::~ListOp(){}
336
337 void ListOp::display_file_entry(const std::string & filename, const Tool::OptParams & opts, FILE* filestream){
338 (void) opts;
339 fputs(filename.c_str(), filestream);
340 fputs("\n",filestream);
341 }
342
343 void ListOp::display_long_file_entry(const std::string & filename, struct stat* st, const Tool::OptParams & opts, FILE* filestream){
344 (void) opts;
345 std::ostringstream ss;
346 ss << Tool::string_from_mode(st->st_mode) << " ";
347 ss << Tool::string_from_size_t(static_cast<size_t>(st->st_nlink),4) << " ";
348 ss << Tool::string_from_size_t(st->st_size, 9) << " ";
349 ss << Tool::string_from_ptime(st->st_mtime) << " ";
350 ss << filename << "\n";
351
352 fputs(ss.str().c_str(), filestream);
353 }
354
355 struct DirEntry {
356 std::string fullURL;
357 std::string path;
358 struct stat st;
359 DirEntry(std::string _fullURL, std::string _path, struct stat _st)
360 : fullURL(_fullURL), path(_path), st(_st) {}
361 };
362
363 struct FileEntry {
364 std::string path;
365 struct stat st;
366 FileEntry(std::string _path, struct stat _st)
367 : path(_path), st(_st) {}
368 };
369
370 int ListOp::executeOp(){
371 DAVIX_DIR* fd = NULL;
372 DavPosix pos(&_c);
373 DavixError* tmp_err=NULL;
374 std::string outputPath;
375 struct stat st;
376 struct dirent* d;
377 unsigned long entry_counter = 0;
378 std::string last_success_entry;
379
380 std::deque< struct DirEntry > dirQueue;
381 std::deque< struct FileEntry > fileQueue;
382
383 // set up first entry
384 if(_target_url.empty()){
385 DavixError::setupError(&tmp_err, "Davix::ListOp", StatusCode::InvalidArgument, " target URL is empty.");
386 }
387
388 if( (fd = pos.opendirpp(&_opts.params, _target_url, &tmp_err)) == NULL){
389 Tool::errorPrint(&tmp_err);
390 return -1;
391 }
392
393 Uri tmp(_target_url);
394 std::string fullpath = tmp.getPath();
395 if(fullpath[0] == '/') fullpath.erase(0,1);
396
397 while( ((d = pos.readdirpp(fd, &st, &tmp_err)) != NULL)){ // if one entry inside a directory fails, the loop exits, the other entires are not processed
398
399 last_success_entry = Uri::join(_target_url, d->d_name);
400 // for each entry, see if it's a directory, if yes, push to dirQueue for further processing
401 if(st.st_mode & S_IFDIR){
402 DAVIX_SLOG(DAVIX_LOG_DEBUG, DAVIX_LOG_CORE, "Directory entry found, pushing {}/ to dirQueue", _target_url+d->d_name);
403 dirQueue.push_back(DirEntry(Uri::join(_target_url, d->d_name)+"/", Uri::join(fullpath, d->d_name), st));
404 }
405 else{
406 fileQueue.push_back(FileEntry(Uri::join(fullpath, d->d_name), st));
407 }
408
409 } // while readdirpp
410
411
412 if(tmp_err){
413 Tool::errorPrint(&tmp_err);
414 std::cerr << std::endl << "Error occurred during listing " << _target_url << " Number of entries processed in current directory: " << entry_counter << ". Continuing..."<< std::endl;
415 std::cerr << std::endl << "Last successful entry is " << last_success_entry << std::endl;
416 }
417
418 entry_counter = 0;
419 pos.closedirpp(fd, NULL);
420
421 {
422 // lock this part so the output is sync'ed
423 // this should also ensure all the child ops are ordered in the taskqueue
424 DavixMutex mutex(_output_mutex);
425
426 for(unsigned int i=0; i < dirQueue.size(); ++i){
427 //push listing op to task queue
428 DAVIX_SLOG(DAVIX_LOG_DEBUG, DAVIX_LOG_CORE, "Adding item to (listing) work queue, target is {}.", dirQueue[i].fullURL);
429 ListOp* l_op = new ListOp(_opts, dirQueue[i].fullURL, _c, _listing_tq, _filestream, _output_mutex);
430 _listing_tq->pushOp(l_op);
431
432 if(_opts.pres_flag & LONG_LISTING_FLAG){
433 display_long_file_entry(dirQueue[i].path, &(dirQueue[i].st), _opts, _filestream);
434 }else{
435 display_file_entry(dirQueue[i].path, _opts, _filestream);
436 }
437 }
438 for(unsigned int i=0; i < fileQueue.size(); ++i){
439 if(_opts.pres_flag & LONG_LISTING_FLAG){
440 display_long_file_entry(fileQueue[i].path, &(fileQueue[i].st), _opts, _filestream);
441 }else{
442 display_file_entry(fileQueue[i].path, _opts, _filestream);
443 }
444 }
445
446 }
447
448 if(tmp_err){
449 Tool::errorPrint(&tmp_err);
450 return -1;
451 }
452 return 0;
453 }
454
455
456 //-------------------------------------------------
457 //---------------------ListppOp----------------------
458 //-------------------------------------------------
459 ListppOp::ListppOp(const Tool::OptParams& opts, std::string target_url, std::string destination_url, Context& c, DavixTaskQueue* tq, DavixTaskQueue* listing_tq) :
460 DavixOp(opts, target_url, destination_url, c)
461 {
462 opType = "LISTPP";
463 _scope = "Davix::DavixOp::ListppOp";
464 _tq = tq;
465 _listing_tq = listing_tq;
466 }
467
468 ListppOp::~ListppOp(){}
469
470 int ListppOp::executeOp(){
471 DAVIX_DIR* fd = NULL;
472 DavPosix pos(&_c);
473 DavixError* tmp_err=NULL;
474 std::string outputPath;
475 struct stat st;
476 struct dirent* d;
477 unsigned long entry_counter=0;
478 std::string last_success_entry;
479
480 std::deque<std::pair<std::string,std::string> > dirQueue;
481 std::deque<std::pair<std::string,std::string> > opQueue;
482
483 // set up first entry
484 if(!_target_url.empty() && !_destination_url.empty()){
485 dirQueue.push_back(std::make_pair(_target_url, _destination_url));
486 }
487 else
488 DavixError::setupError(&tmp_err, "Davix::ListppOp", StatusCode::InvalidArgument, " target or destination URL is empty.");
489
490 if( (fd = pos.opendirpp(&_opts.params, dirQueue.front().first, &tmp_err)) == NULL){
491 Tool::errorPrint(&tmp_err);
492
493 return -1;
494 }
495
496 // dir opened successfully, create local dir
497 Tool::mkdirP(dirQueue.front().second, false);
498
499 while( ((d = pos.readdirpp(fd, &st, &tmp_err)) != NULL)){ // if one entry inside a directory fails, the loop exits, the other entires are not processed
500
501 last_success_entry = Uri::join(dirQueue.front().first, d->d_name);
502 // for each entry, see if it's a directory, if yes, push to dirQueue for further processing
503 if(st.st_mode & S_IFDIR){
504 DAVIX_SLOG(DAVIX_LOG_DEBUG, DAVIX_LOG_CORE, "Directory entry found, pushing {}/ to dirQueue", dirQueue.front().first+d->d_name);
505 dirQueue.push_back(std::make_pair(Uri::join(dirQueue.front().first, d->d_name)+"/",Uri::join(dirQueue.front().second, d->d_name)));
506 }
507 else if(!(st.st_mode & S_IFDIR)){
508 //if we spend too long in here, server will likely close the connection mid-readdirpp, need to get all the entries quickly before processing them
509 opQueue.push_back(std::make_pair(Uri::join(dirQueue.front().first, d->d_name), Uri::join(dirQueue.front().second, d->d_name)));
510 entry_counter++;
511 }
512 } // while readdirpp
513
514 if(tmp_err){
515 Tool::errorPrint(&tmp_err);
516 std::cerr << std::endl << "Error occurred during listing " << dirQueue.front().first << " Number of entries processed in current directory: " << entry_counter << ". Continuing..."<< std::endl;
517 std::cerr << std::endl << "Last successful entry is " << last_success_entry << std::endl;
518 }
519
520 pos.closedirpp(fd, NULL);
521 dirQueue.pop_front();
522
523 for(unsigned int i=0; i < dirQueue.size(); ++i){
524 //push listing op to task queue
525 DAVIX_SLOG(DAVIX_LOG_DEBUG, DAVIX_LOG_CORE, "Adding item to (listing) work queue, target is {} and destination is {}.", dirQueue[i].first, dirQueue[i].second);
526 ListppOp* l_op = new ListppOp(_opts, (dirQueue[i].first), (dirQueue[i].second), _c, _tq, _listing_tq);
527 _listing_tq->pushOp(l_op);
528 }
529
530 for(unsigned int i=0; i < opQueue.size(); ++i){
531 //push op to task queue
532 DAVIX_SLOG(DAVIX_LOG_DEBUG, DAVIX_LOG_CORE, "Adding item to (get) work queue, target is {} and destination is {}.", opQueue[i].first, opQueue[i].second);
533
534 GetOp* op = new GetOp(_opts, (opQueue[i].first), (opQueue[i].second), _c);
535 _tq->pushOp(op);
536 }
537
538 if(tmp_err){
539 Tool::errorPrint(&tmp_err);
540 return -1;
541 }
542 return 0;
543 }
544
545
546 } // namespace Davix