Debian Codegraph @

src/tools/davix_tool_get_main.cpp

raw
1 /*
2 * This File is part of Davix, The IO library for HTTP based protocols
3 * Copyright (C) CERN 2013
4 * Author: Adrien Devresse <[email protected]>
5 *
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2.1 of the License, or (at your option) any later version.
10 *
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, write to the Free Software
18 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19 *
20 */
21
22 #include <davix_internal.hpp>
23 #include <davix.hpp>
24 #include <tools/davix_tool_params.hpp>
25 #include <tools/davix_tool_util.hpp>
26 #include <tools/davix_taskqueue.hpp>
27 #include <tools/davix_op.hpp>
28 #include <tools/davix_thread_pool.hpp>
29 #include <utils/davix_logger_internal.hpp>
30
31
32 // @author : Devresse Adrien
33 // main file for davix-get operation
34
35
36 using namespace Davix;
37 using namespace std;
38
39 #define READ_BLOCK_SIZE 4096
40
41
42 const std::string scope_get = "Davix::Tools::davix-get";
43
44 std::string get_base_get_options(){
45 return " Get Options:\n"
46 "\t--accepted-retry: Number of retries upon receiving 202-Accepted. default: 180\n"
47 "\t--accepted-retry-delay: Time in seconds to wait between 202-Accepted retries. default: 10\n"
48 "\t-r NUMBER_OF_THREADS: Get directories and their contents recursively.\n";
49 }
50
51 static std::string help_msg(const std::string &cmd_path){
52 std::string help_msg = fmt::format("Usage : {} ", cmd_path);
53 help_msg += Tool::get_get_description_options();
54 help_msg += Tool::get_common_options();
55 help_msg += get_base_get_options();
56
57 return help_msg;
58 }
59
60 int get_output_get_fstream(const Tool::OptParams & opts, const std::string & scope, std::string uri, DavixError** err){
61 int fd = -1;
62 if(uri.empty() == false){
63 if((fd = open(uri.c_str(), O_WRONLY | O_TRUNC | O_CREAT, 0777)) <0 ){
64 davix_errno_to_davix_error(errno, scope, std::string("for destination file ").append(opts.output_file_path), err);
65 return -1;
66 }
67 }else{
68 fd = dup(STDOUT_FILENO);
69 }
70 return fd;
71 }
72
73 static int excute_get(Context& c, const Tool::OptParams & opts, int out_fd, std::string uri, DavixError** err){
74 int ret;
75 DavFile f(c, uri);
76 ret = f.getToFd(&opts.params, out_fd, err);
77 return (ret >= 0)?0:-1;
78 }
79
80 static int populateTaskQueue(Context& c, const Tool::OptParams & opts, std::string uri, DavixTaskQueue* tq, DavixTaskQueue* listing_tq, DavixError** err ){
81 DAVIX_DIR* fd = NULL;
82 DavPosix pos(&c);
83 DavixError* tmp_err=NULL;
84 std::string outputPath;
85 struct stat st;
86 struct dirent* d;
87 int entry_counter=0;
88 std::string last_success_entry;
89
90 std::deque<std::pair<std::string,std::string> > dirQueue;
91 std::deque<std::pair<std::string,std::string> > opQueue;
92
93 outputPath = opts.output_file_path;
94
95 // set up first entry
96 if(!uri.empty()){
97 // if output path is not specified, default to current directory
98 if(outputPath.empty()){
99 std::deque<std::string> dirVec;
100
101 std::string tmp = Uri(uri).getPath();
102 if(tmp == "/"){ // no output path and no token in url to use as default, fallback to ./temp
103 outputPath = "./temp";
104 }
105 else{
106 Tool::tokeniseUrl(Uri(uri).getPath(), dirVec);
107 outputPath = "./"+dirVec.back();
108 }
109 }
110 dirQueue.push_back(std::make_pair(uri, outputPath));
111 }
112 else
113 DavixError::setupError(&tmp_err, "Davix::Tool::Get", StatusCode::InvalidArgument, " target URL is empty.");
114
115 if( (fd = pos.opendirpp(&opts.params, dirQueue.front().first, &tmp_err)) == NULL){
116 Tool::errorPrint(&tmp_err);
117
118 // if the "root" level cannot be found, there is nothing we can do, just quit
119 return -1;
120 }
121
122 // dir opened successfully, create local dir
123 Tool::mkdirP(dirQueue.front().second, false);
124
125 //if S3, remove last token of url to accommodate duplicate token in d_name
126 if (opts.params.getProtocol() == RequestProtocol::AwsS3){
127 dirQueue.front().first = "s3://"+(Uri(dirQueue.front().first).getHost())+"/";
128 }
129
130 while( ((d = pos.readdirpp(fd, &st, &tmp_err)) != NULL)){ // if one entry inside a directory fails, the loop exits, the other entires are not processed
131
132 last_success_entry = dirQueue.front().first+d->d_name;
133 // for each entry, do a stat to see if it's a directory, if yes, push to dirQueue for further processing
134 if(st.st_mode & S_IFDIR){
135 DAVIX_SLOG(DAVIX_LOG_DEBUG, DAVIX_LOG_CORE, "Directory entry found, pushing {}/ to dirQueue", dirQueue.front().first+d->d_name);
136 dirQueue.push_back(std::make_pair(Uri::join(dirQueue.front().first, d->d_name)+"/", Uri::join(dirQueue.front().second, d->d_name)));
137 }
138 else if(!(st.st_mode & S_IFDIR)){
139 //if we spend too long in here, server will likely close the connection mid-readdirpp, need to get all the entries quickly before processing them
140 opQueue.push_back(std::make_pair(Uri::join(dirQueue.front().first, d->d_name), Uri::join(dirQueue.front().second, d->d_name)));
141 }
142 entry_counter++;
143 if(!opts.debug)
144 Tool::batchTransferMonitor(dirQueue.front().first, "Crawling", entry_counter, 0);
145 } // while readdirpp
146
147 if(tmp_err){
148 Tool::errorPrint(&tmp_err);
149 cerr << endl << "Error occurred during listing " << dirQueue.front().first << " Number of entries processed in current directory: " << entry_counter << ". Continuing..."<< endl;
150 cerr << endl << "Last successful entry is " << last_success_entry << endl;
151 }
152
153 entry_counter = 0;
154
155 pos.closedirpp(fd, NULL);
156 dirQueue.pop_front();
157
158 int num_of_ops = opQueue.size();
159 int num_listing_ops = dirQueue.size();
160
161 // if endpoint is S3 then there is no need to crawl namespace recursively, since it's flat
162 if (opts.params.getProtocol() != RequestProtocol::AwsS3){
163 for(unsigned int i=0; i < dirQueue.size(); ++i){
164 //push listing op to task queue
165 DAVIX_SLOG(DAVIX_LOG_DEBUG, DAVIX_LOG_CORE, "Adding item to (listing) work queue, target is {} and destination is {}.", dirQueue[i].first, dirQueue[i].second);
166
167 entry_counter++;
168
169 if(!opts.debug)
170 Tool::batchTransferMonitor(dirQueue[i].first, "Populating (listing) task queue for", entry_counter, num_listing_ops);
171
172 ListppOp* l_op = new ListppOp(opts, (dirQueue[i].first), (dirQueue[i].second), c, tq, listing_tq);
173 listing_tq->pushOp(l_op);
174 }
175 entry_counter = 0;
176 }
177
178 for(unsigned int i=0; i < opQueue.size(); ++i){
179 //push op to task queue
180 DAVIX_SLOG(DAVIX_LOG_DEBUG, DAVIX_LOG_CORE, "Adding item to (get) work queue, target is {} and destination is {}.", opQueue[i].first, opQueue[i].second);
181
182 entry_counter++;
183
184 if(!opts.debug)
185 Tool::batchTransferMonitor(opQueue[i].first, "Populating (get) task queue for", entry_counter, num_of_ops);
186
187 GetOp* op = new GetOp(opts, (opQueue[i].first), (opQueue[i].second), c);
188 tq->pushOp(op);
189 }
190
191 if(tmp_err){
192 DavixError::propagateError(err, tmp_err);
193 return -1;
194 }
195 return 0;
196 }
197
198
199 static int preGetCheck(Tool::OptParams & opts, DavixError** err ) {
200 Context c;
201 configureContext(c, opts);
202 File f(c, opts.vec_arg[0]);
203 struct stat st;
204 DavixError* tmp_err=NULL;
205
206
207 // Try to understand what to do
208 // If the URL is for plain HTTP then we don't want to stat
209 bool justgetfile = true;
210
211 if ( opts.vec_arg[0].compare(0, 4, "http") != 0 ) {
212 int r = f.stat(&opts.params, &st, &tmp_err);
213 if (r && tmp_err) {
214 DavixError::propagateError(err, tmp_err);
215 return -1;
216 }
217
218 if ( !r && (st.st_mode & S_IFDIR) && opts.params.getRecursiveMode()) justgetfile = false;
219 }
220
221 if (justgetfile) {
222 // target resource is a file or request protocol is http, just get it normally
223 int out_fd= -1;
224 if( ( (out_fd = get_output_get_fstream(opts, scope_get, opts.output_file_path, &tmp_err)) > 0)
225 && (Tool::configureMonitorCB(opts, Transfer::Read)) == 0) {
226
227 excute_get(c, opts, out_fd, opts.vec_arg[0], &tmp_err);
228 Tool::flushFinalLineShell(out_fd);
229 close(out_fd);
230 }
231 }
232 else {
233
234 std::string url(opts.vec_arg[0]);
235
236 if (url[url.size()-1] != '/')
237 url += '/';
238
239 // if protocol is S3, set listing mode to SemiHierarchical, we want to get every object under the same prefix in one go
240 if (opts.params.getProtocol() == RequestProtocol::AwsS3){
241 opts.params.setS3ListingMode(S3ListingMode::SemiHierarchical);
242 // unfortunately s3 defaults max-keys to 1000 and doesn't provide a way to disable the cap, set to large number
243 opts.params.setS3MaxKey(999999999);
244 }
245
246 DavixTaskQueue tq; // for get ops
247 DavixTaskQueue listing_tq; // for listing ops
248
249 // create threadpool instance
250 DAVIX_SLOG(DAVIX_LOG_DEBUG, DAVIX_LOG_CORE, "Creating threadpool");
251 DavixThreadPool tp(&tq, opts.threadpool_size);
252 DavixThreadPool listing_tp(&listing_tq, opts.threadpool_size);
253
254 populateTaskQueue(c, opts, url, &tq, &listing_tq, &tmp_err);
255
256 // if both task queues are empty, then all work is done, stop workers. Otherwise wait.
257 do{
258 sleep(2);
259 }while(!tq.isEmpty() || !listing_tq.isEmpty() || !tp.allThreadsIdle() || !listing_tp.allThreadsIdle());
260
261 listing_tp.shutdown();
262 tp.shutdown();
263 Tool::flushFinalLineShell(STDOUT_FILENO);
264 }
265
266 if(tmp_err){
267 DavixError::propagateError(err, tmp_err);
268 return -1;
269 }
270 return 0;
271 }
272
273
274 int main(int argc, char** argv){
275 int retcode=-1;
276 Tool::OptParams opts;
277 DavixError* tmp_err=NULL;
278 opts.help_msg = help_msg(argv[0]);
279
280 if( (retcode= Tool::parse_davix_get_options(argc, argv, opts, &tmp_err)) ==0
281 && (retcode = Tool::configureAuth(opts)) == 0 && checkProtocolSanity(opts, opts.vec_arg[0], &tmp_err)){
282 retcode = preGetCheck(opts, &tmp_err);
283 }
284 Tool::errorPrint(&tmp_err);
285
286 return retcode;
287 }