Debian Codegraph @

src/tools/davix_tool_put_main.cpp

raw
1 /*
2 * This File is part of Davix, The IO library for HTTP based protocols
3 * Copyright (C) CERN 2013
4 * Author: Adrien Devresse <[email protected]>
5 *
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2.1 of the License, or (at your option) any later version.
10 *
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, write to the Free Software
18 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19 *
20 */
21
22 #include <davix.hpp>
23 #include <davix_internal.hpp>
24 #include <tools/davix_tool_params.hpp>
25 #include <tools/davix_tool_util.hpp>
26 #include <tools/davix_taskqueue.hpp>
27 #include <tools/davix_op.hpp>
28 #include <tools/davix_thread_pool.hpp>
29 #include <utils/davix_logger_internal.hpp>
30 #include <cstdio>
31
32
33 // @author : Devresse Adrien
34 // main file for davix-put operation
35
36
37 using namespace Davix;
38 using namespace std;
39
40 #define READ_BLOCK_SIZE 4096
41
42
43 const std::string scope_put = "Davix::Tools::davix-put";
44
45 std::string get_base_put_options(){
46 return " Put Options:\n"
47 "\t-r NUMBER_OF_THREADS: Upload directories and their contents recursively\n"
48 "\t--no-100-continue Never ask for a 100-Continue from the server (some do not support it)\n";
49 }
50
51 static std::string help_msg(const std::string & cmd_path){
52 std::string help_msg = fmt::format("Usage : {} ", cmd_path);
53 help_msg += Tool::get_put_description_options();
54 help_msg += Tool::get_common_options();
55 help_msg += get_base_put_options();
56
57 return help_msg;
58 }
59
60
61 static int execute_put(Context& c, const Tool::OptParams & opts, int fd, DavixError** err){
62 const std::string & src_file = opts.input_file_path;
63 const std::string & dst_file = opts.vec_arg[1];
64
65 TRY_DAVIX{
66 DavFile f(c, dst_file);
67 struct stat st;
68 if( fstat(fd, &st) != 0){
69 errno_to_davix_exception(errno, scope_put, std::string("for source file ").append(src_file));
70 }
71 if( S_ISREG(st.st_mode) && (st.st_size >= 0)){
72 f.put(&opts.params, fd, static_cast<dav_size_t>(st.st_size));
73 return 0;
74 }
75 throw DavixException(scope_put, StatusCode::SystemError, std::string(src_file).append("is not a valid regular file"));
76 }CATCH_DAVIX(err);
77 return -1;
78 }
79
80
81 static int tryMakeCollection(Context& c, const Tool::OptParams & opts, std::string dst_path){
82 DavixError* err=NULL;
83 DavFile f(c,dst_path);
84 f.makeCollection(&opts.params, &err);
85
86 if(err){
87 if(err->getStatus() == StatusCode::FileExist){
88 std::cerr << std::endl << scope_put << " " << dst_path << " already exists, continuing..." << std::endl;
89 DavixError::clearError(&err);
90 }
91 else{
92 Tool::errorPrint(&err);
93 return -1;
94 }
95 }
96 return 0;
97 }
98
99 static int populateTaskQueue(Context& c, const Tool::OptParams & opts, std::string src_path, std::string dst_path, DavixTaskQueue* tq, DavixError** err ){
100 struct stat st;
101 DIR* dp;
102 struct dirent *de;
103 int ret = -1;
104 int entry_counter = 0;
105
106 if((dp = opendir(src_path.c_str()) ) == NULL){
107 DavixError* tmp_err=NULL;
108 davix_errno_to_davix_error(errno, scope_put, std::string("for source file ").append(src_path), &tmp_err);
109 Tool::errorPrint(&tmp_err);
110 return -1;
111 }
112
113 // process entries recursively
114 while ((de = readdir(dp)) != NULL) {
115 if((strcmp(de->d_name, ".")) && (strcmp(de->d_name, ".."))){ // ignore "." and ".." entries
116 if(stat(((src_path+de->d_name).c_str()), &st) == 0){ // check if entry is file or directory
117 if(S_ISDIR(st.st_mode)){ // is directory
118 if(opts.params.getProtocol() != RequestProtocol::AwsS3){ // if protocol is S3, don't need make collection (not hierarchical)
119 ret = tryMakeCollection(c, opts, Uri::join(dst_path, de->d_name));
120 if(ret <0)
121 return ret;
122 }
123 ret = populateTaskQueue(c, opts, Uri::join(src_path, de->d_name)+"/", Uri::join(dst_path, de->d_name), tq, err);
124 }
125 else if(S_ISREG(st.st_mode) && st.st_size > 0){
126 //push op to task queue
127 DAVIX_SLOG(DAVIX_LOG_DEBUG, DAVIX_LOG_CORE, "Adding item to work queue, source is {} and destination is {}.", src_path+de->d_name, dst_path+"/"+de->d_name);
128 PutOp* op = new PutOp(opts, Uri::join(src_path, de->d_name), Uri::join(dst_path, de->d_name), static_cast<dav_size_t>(st.st_size), c);
129 tq->pushOp(op);
130 entry_counter++;
131 }
132 }
133 else{
134 errno_to_davix_exception(errno, scope_put, std::string("for source file ").append(src_path));
135 return -1;
136 }
137 }
138 if(!opts.debug)
139 Tool::batchTransferMonitor(src_path, "Populating task queue for", entry_counter, 0);
140 }
141 closedir(dp);
142 return (err)?(-1):0;
143 }
144
145
146 static int prePutCheck(Tool::OptParams & opts, DavixError** err){
147 struct stat st;
148 int ret = -1;
149 Context c;
150 configureContext(c, opts);
151
152 if((ret = stat(opts.input_file_path.c_str(), &st)) != 0){
153 DavixError* tmp_err=NULL;
154 davix_errno_to_davix_error(errno, scope_put, std::string("for source file ").append(opts.input_file_path), &tmp_err);
155 DavixError::propagateError(err, tmp_err);
156 return -1;
157 }
158
159 if((st.st_mode & S_IFDIR) && opts.params.getRecursiveMode()){
160 std::string src_path(opts.input_file_path);
161
162 if(src_path[src_path.size()] != '/')
163 src_path += '/';
164
165 DavixTaskQueue tq;
166
167 // create threadpool instance
168 DAVIX_SLOG(DAVIX_LOG_DEBUG, DAVIX_LOG_CORE, "Creating threadpool");
169 DavixThreadPool tp(&tq, opts.threadpool_size);
170
171 if(opts.params.getProtocol() != RequestProtocol::AwsS3 || opts.params.getProtocol() != RequestProtocol::Swift){ // if protocol is S3 or Swift, don't need make collection (not hierarchical)
172 ret = tryMakeCollection(c, opts, opts.vec_arg[1]);
173 if(ret <0)
174 return ret;
175 }
176
177 std::string url(opts.vec_arg[1]);
178 if (url[url.size()-1] == '/')
179 url.erase(url.size() - 1);
180
181 ret = populateTaskQueue(c, opts, src_path, url, &tq, err);
182
183 // if task queue is empty, then all work is done, stop workers. Otherwise wait.
184 while(!tq.isEmpty()){
185 sleep(2);
186 }
187 tp.shutdown();
188 Tool::flushFinalLineShell(STDOUT_FILENO);
189 }
190 else{ // single file to upload, process it normally
191 int fd_in = -1;
192 if(((fd_in = Tool::getInFd(opts, scope_put, err)) > 0)
193 && (Tool::configureMonitorCB(opts, Transfer::Write)) == 0){
194 ret = execute_put(c, opts, fd_in, err);
195 close(fd_in);
196 }
197 }
198 return ret;
199 }
200
201
202 int main(int argc, char** argv){
203 int retcode=-1;
204 Tool::OptParams opts;
205 DavixError* tmp_err=NULL;
206 opts.help_msg = help_msg(argv[0]);
207
208 if(( (retcode = Tool::parse_davix_put_options(argc, argv, opts, &tmp_err)) == 0)
209 && ((retcode = Tool::configureAuth(opts)) == 0) && checkProtocolSanity(opts, opts.vec_arg[1], &tmp_err)){
210 retcode = prePutCheck(opts, &tmp_err);
211 }
212 Tool::errorPrint(&tmp_err);
213 return retcode;
214 }