Debian Codegraph @

src/tools/davix_tool_rm_main.cpp

raw
1 /*
2 * This File is part of Davix, The IO library for HTTP based protocols
3 * Copyright (C) CERN 2013
4 * Author: Adrien Devresse <[email protected]>
5 *
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2.1 of the License, or (at your option) any later version.
10 *
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, write to the Free Software
18 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19 *
20 */
21
22 #include <davix_internal.hpp>
23 #include <davix.hpp>
24 #include <tools/davix_tool_params.hpp>
25 #include <tools/davix_tool_util.hpp>
26 #include <tools/davix_taskqueue.hpp>
27 #include <tools/davix_op.hpp>
28 #include <tools/davix_thread_pool.hpp>
29 #include <utils/davix_logger_internal.hpp>
30 #include <cstdio>
31 #include <fstream>
32
33 // @author : Devresse Adrien
34 // main file for davix low level cmd line tool
35
36
37 using namespace Davix;
38 using namespace std;
39
40 #define READ_BLOCK_SIZE 4096
41
42
43 const std::string scope_main = "Davix::Tools::davix-rm";
44
45 const std::string openTag_req = "<Delete>";
46 const std::string closeTag_req = "</Delete>";
47 const std::string openTags_entry = "<Object><Key>";
48 const std::string closeTags_entry = "</Key></Object>";
49
50 std::string get_base_rm_options(){
51 return " Delete Options:\n"
52 "\t-r NUMBER_OF_THREADS: Remove contents recursively, also works on S3 objects for storage systems that supports multi-objects deletion.\n"
53 "\t-n NUMBER_OF_KEYS: Number of object keys to include in each delete request, S3 supports up to 1000, but request will likely timeout. Default: 20\n";
54 }
55 static std::string help_msg(const std::string & cmd_path){
56 std::string help_msg = fmt::format("Usage : {} ", cmd_path);
57 help_msg += Tool::get_base_description_options();
58 help_msg += Tool::get_common_options();
59 help_msg += get_base_rm_options();
60
61 return help_msg;
62 }
63
64 static int populateTaskQueue(Context& c, const Tool::OptParams & opts, std::string uri, DavixTaskQueue* tq, DavixError** err ){
65 DAVIX_DIR* fd = NULL;
66 DavPosix pos(&c);
67 DavixError* tmp_err=NULL;
68 struct stat st;
69 struct dirent* d;
70 int counter = 0;
71 int entry_counter = 0;
72 std::string protocol, host;
73 std::string buffer(openTag_req);
74
75 if( (fd = pos.opendirpp(&opts.params, opts.vec_arg[0], &tmp_err)) == NULL){
76 DavixError::propagateError(err, tmp_err);
77 return -1;
78 }
79
80 if(opts.vec_arg[0].compare(2,1,"s") == 0){
81 protocol = "s3s://";
82 }
83 else{
84 protocol = "s3://";
85 }
86
87 host = protocol + Uri(uri).getHost();
88
89 while( (d = pos.readdirpp(fd, &st, &tmp_err)) != NULL ){
90 // for every entry
91
92 //TODO: unless user has turned on rr mode, skip entries that end with a '/'?
93 //that way we can provide a way to just delete files inside the directory and not the other directories
94
95 // format to xml and push to buffer string, for every n entries, create a DeleteOp(PostRequest)
96 std::string entry(d->d_name);
97
98 // for each entry, <Object><Key>entry</Key></Object>
99 buffer += openTags_entry + entry + closeTags_entry;
100 counter++;
101 entry_counter++;
102
103 // although the S3 API supports up to 1000 keys per request, in practice this will most likely results in a 504 gateway timeout
104 if(counter == opts.s3_delete_per_request){ // default is set to 20, just to be safe
105 buffer += closeTag_req;
106
107
108 // push op to task queue
109 DAVIX_SLOG(DAVIX_LOG_DEBUG, DAVIX_LOG_CORE, "Adding item to work queue, target is {}.", host);
110
111 DeleteOp* op = new DeleteOp(opts, host, c, buffer);
112 tq->pushOp(op);
113
114 counter = 0;
115 buffer.clear();
116 buffer = openTag_req;
117 }
118
119 if(!opts.debug)
120 Tool::batchTransferMonitor(uri, "Multi-objects delete ", entry_counter, 0);
121 }
122 // check if there is still anything in buffer for remainder entries
123 if(!buffer.empty()){
124 buffer += closeTag_req;
125
126 // push op to task queue
127 DAVIX_SLOG(DAVIX_LOG_DEBUG, DAVIX_LOG_CORE, "Adding item to work queue, target is {}.", host);
128
129 DeleteOp* op = new DeleteOp(opts, host, c, buffer);
130 tq->pushOp(op);
131 }
132
133 pos.closedirpp(fd, NULL);
134 return 0;
135 }
136
137
138 static int preDeleteCheck(Tool::OptParams & opts, DavixError** err ) {
139 Context c;
140 configureContext(c, opts);
141 DavPosix f(&c);
142 struct stat st;
143 DavixError* tmp_err=NULL;
144
145 // see if the requested url actually points to a directory or not
146 int r = f.stat(&opts.params, opts.vec_arg[0], &st, &tmp_err);
147 if (tmp_err) {
148 DavixError::propagateError(err, tmp_err);
149 return -1;
150 }
151 if (r && !(st.st_mode & S_IFDIR)){
152 DavixError::setupError(&tmp_err, scope_main, StatusCode::IsNotADirectory, "Failed to delete, target is not a directory.");
153 DavixError::propagateError(err, tmp_err);
154 return -1;
155 }
156
157 DavixTaskQueue tq;
158
159 // create threadpool instance
160 DAVIX_SLOG(DAVIX_LOG_DEBUG, DAVIX_LOG_CORE, "Creating threadpool");
161 DavixThreadPool tp(&tq, opts.threadpool_size);
162
163 // TODO two modes, one to delete "recursively, one for depth of 1. can probbaly do it in readdirpp
164
165 // set listing mode to SemiHierarchical, we want to get every object under the same prefix in one go
166 opts.params.setS3ListingMode(S3ListingMode::SemiHierarchical);
167 // unfortunately s3 defaults max-keys to 1000 and doesn't provide a way to disable the cap, set to large number
168 opts.params.setS3MaxKey(999999999);
169
170 std::string url(opts.vec_arg[0]);
171
172 if (url[url.size()-1] != '/')
173 url += '/';
174
175 populateTaskQueue(c, opts, url, &tq, &tmp_err);
176
177 // if task queue is empty, then all work is done, stop workers. Otherwise wait.
178 while(!tq.isEmpty()){
179 sleep(2);
180 }
181 tp.shutdown();
182 Tool::flushFinalLineShell(STDOUT_FILENO);
183
184 if(tmp_err){
185 DavixError::propagateError(err, tmp_err);
186 return -1;
187 }
188 return 0;
189 }
190
191
192 int main(int argc, char** argv){
193 int retcode=-1;
194 Tool::OptParams opts;
195 DavixError* tmp_err=NULL;
196 opts.help_msg = help_msg(argv[0]);
197
198 if( (retcode= Tool::parse_davix_rm_options(argc, argv, opts, &tmp_err)) ==0){
199 if( (retcode = Tool::configureAuth(opts)) == 0){
200 if(checkProtocolSanity(opts, opts.vec_arg[0], &tmp_err)) {
201 if ( opts.params.getAzureKey().size() != 0) {
202 opts.params.setProtocol(RequestProtocol::Azure);
203 }else if (opts.vec_arg[0].compare(0,4,"http") ==0){
204 opts.params.setProtocol(RequestProtocol::Http);
205 }else if( opts.vec_arg[0].compare(0,2,"s3") ==0){
206 opts.params.setProtocol(RequestProtocol::AwsS3);
207 }else if ( opts.vec_arg[0].compare(0, 3,"dav") ==0){
208 opts.params.setProtocol(RequestProtocol::Webdav);
209 }
210
211 // only s3 needs special treatment, WebDAV delete on collection already works
212 if((opts.params.getProtocol() == RequestProtocol::AwsS3) && (opts.params.getRecursiveMode())){
213 preDeleteCheck(opts, &tmp_err);
214 }
215 else{
216 Context c;
217 configureContext(c, opts);
218
219 DavFile f(c,opts.vec_arg[0]);
220 f.deletion(&opts.params, &tmp_err);
221 }
222 if(tmp_err != NULL) retcode = 1;
223 }
224 }
225 }
226 Tool::errorPrint(&tmp_err);
227 return retcode;
228 }