Debian Codegraph @

src/fileops/davix_reliability_ops.cpp

raw
1 /*
2 * This File is part of Davix, The IO library for HTTP based protocols
3 * Copyright (C) CERN 2013
4 * Author: Adrien Devresse <[email protected]>
5 *
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2.1 of the License, or (at your option) any later version.
10 *
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, write to the Free Software
18 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19 *
20 */
21
22 #include <mutex>
23 #include <davix_internal.hpp>
24 #include "davix_reliability_ops.hpp"
25
26 #include <utils/stringutils.hpp>
27 #include <utils/davix_logger_internal.hpp>
28 #include <xml/metalinkparser.hpp>
29 #include "libs/alibxx/crypto/base64.hpp"
30
31
32 namespace Davix{
33
34
35 using namespace StrUtil;
36
37 typedef std::function< dav_ssize_t (IOChainContext &)> FuncIO;
38 typedef std::function< StatInfo & (IOChainContext &) > FuncStatInfo;
39
40
41 static bool metalink_support_disabled=false;
42 static std::once_flag metalink_once;
43
44
45 void propagateNonRecoverableExceptions(DavixException & e){
46 /// Forward redirections and other error we don't want to recover
47 if(e.code() == StatusCode::RedirectionNeeded
48 || e.code() == StatusCode::OperationTimeout){
49 throw e;
50 }
51 }
52
53 void metalink_check(){
54 metalink_support_disabled = (getenv("DAVIX_DISABLE_METALINK") != NULL);
55 }
56
57 static bool isMetalinkDisabled(const RequestParams* params){
58 std::call_once(metalink_once, metalink_check);
59 return (params != NULL && params->getMetalinkMode() == MetalinkMode::Disable) || metalink_support_disabled;
60 }
61
62
63 template<class Executor, class ReturnType>
64 ReturnType metalinkTryReplicas(HttpIOChain & chain, IOChainContext & io_context, Executor fun){
65 std::vector<File> replicas;
66
67 // check if we expired
68 io_context.checkTimeout();
69 // get all replicas from Metalink
70 chain.getReplicas(io_context, replicas);
71 for(std::vector<File>::iterator it = replicas.begin();it != replicas.end(); ++it){
72 IOChainContext internal_context(io_context._context, it->getUri(), io_context._reqparams);
73 internal_context.fdHandler = io_context.fdHandler;
74
75 try{
76 return fun(internal_context);
77 }catch(DavixException & replica_error){
78 DAVIX_SLOG(DAVIX_LOG_VERBOSE, DAVIX_LOG_CHAIN, "Fail access to replica {}: {}", it->getUri(), replica_error.what());
79 }catch(...){
80 DAVIX_SLOG(DAVIX_LOG_VERBOSE, DAVIX_LOG_CHAIN, "Fail access to replica: Unknown Error");
81 }
82
83 io_context.fdHandler = internal_context.fdHandler;
84 // check timeout again between two iterations
85 io_context.checkTimeout();
86 }
87 throw DavixException(davix_scope_io_buff(), StatusCode::InvalidServerResponse, "Impossible to access any of the replicas with success");
88 }
89
90
91 template<class Executor, class ReturnType>
92 ReturnType metalinkExecutor(HttpIOChain & chain, IOChainContext & io_context, Executor fun){
93 // if disabled, do nothing
94 if(isMetalinkDisabled(io_context._reqparams)){
95 return fun(io_context);
96 }
97
98 try{
99 // Execute operation
100 return fun(io_context);
101 }catch(DavixException & e){
102
103 propagateNonRecoverableExceptions(e);
104
105 DAVIX_SLOG(DAVIX_LOG_VERBOSE, DAVIX_LOG_CHAIN, "Could not execute operation on {}, error {}", io_context._uri.getString(), e.what());
106 DAVIX_SLOG(DAVIX_LOG_VERBOSE, DAVIX_LOG_CHAIN, "Try to Recover with Metalink...");
107
108 try{
109 return metalinkTryReplicas<Executor, ReturnType>(chain, io_context, fun);
110 }catch(DavixException & metalink_error){
111 DAVIX_SLOG(DAVIX_LOG_VERBOSE, DAVIX_LOG_CHAIN, "Impossible to Recover with Metalink: {}", metalink_error.what());
112 }catch(...){
113 DAVIX_SLOG(DAVIX_LOG_VERBOSE, DAVIX_LOG_CHAIN, "Impossible to Recover with Metalink: Unknown Error");
114 }
115 throw e;
116 }
117 }
118
119
120
121 int davix_metalink_header_parser(const std::string & header_key, const std::string & header_value,
122 const Uri & u_original,
123 Uri & metalink){
124 DAVIX_SLOG(DAVIX_LOG_TRACE, DAVIX_LOG_CHAIN, "Parse headers for metalink {} {}", header_key, header_value);
125
126 if(compare_ncase(header_key, "Link") ==0 && header_value.find("application/metalink") != std::string::npos){
127 std::string::const_iterator it1, it2;
128 if( ( it1 = std::find(header_value.begin(), header_value.end(), '<')) != header_value.end()
129 && ( it2 = std::find(it1, header_value.end(), '>')) != header_value.end()){
130 std::string metalink_url(it1+1, it2);
131 metalink = Uri::fromRelativePath(u_original, trim(metalink_url));
132 if(metalink.getStatus() == StatusCode::OK){
133 DAVIX_SLOG(DAVIX_LOG_TRACE, DAVIX_LOG_CHAIN, "Valid metalink URI found {}", metalink.getString());
134 return 1;
135 }
136
137 }
138 }
139 return 0;
140 }
141
142 bool davix_metalink_header_content_type(const std::string & header_key, const std::string & header_value){
143 return (compare_ncase(header_key, "Content-type") ==0 && header_value.find("application/metalink") !=std::string::npos);
144 }
145
146 int davix_get_metalink_url( Context & c, const Uri & uri,
147 const RequestParams & _params, Uri & metalink){
148 DavixError* tmp_err = NULL;
149 RequestParams params(_params);
150 // don't follow redirect, we need headers
151 params.setTransparentRedirectionSupport(false);
152 HeadRequest req(c, uri, &tmp_err);
153 req.setParameters(params);
154 req.addHeaderField("Accept", "application/metalink4+xml");
155
156
157 DAVIX_SLOG(DAVIX_LOG_TRACE, DAVIX_LOG_CHAIN, "Executing head query to {} for Metalink file", uri.getString());
158
159
160 if(tmp_err != NULL || (req.executeRequest(&tmp_err) <0)) {
161 if (tmp_err)
162 throw DavixException(davix_scope_meta(), tmp_err->getStatus(), tmp_err->getErrMsg());
163 else
164 throw DavixException(davix_scope_meta(), Davix::StatusCode::UnknownError, "Unknown error");
165 }
166
167 HeaderVec headers;
168 req.getAnswerHeaders(headers);
169 for(HeaderVec::iterator it = headers.begin(); it != headers.end(); it++){
170 if( davix_metalink_header_parser(it->first, it->second, uri, metalink) > 0)
171 return 1;
172
173 if(davix_metalink_header_content_type(it->first, it->second)){
174 // is a metalink content type, get it
175 metalink = uri;
176 return 1;
177 }
178 }
179
180
181
182
183 return 0;
184 }
185
186
187 int davix_file_get_metalink_to_vfile(Context & c, const Uri & metalink_uri,
188 const RequestParams & _params, std::vector<DavFile> & vec){
189 DavixError * tmp_err=NULL;
190 GetRequest req(c, metalink_uri, &tmp_err);
191 MetalinkParser parser(c, vec);
192
193 req.setParameters(_params);
194 req.addHeaderField("Accept", "application/metalink4+xml");
195
196 DAVIX_SLOG(DAVIX_LOG_TRACE, DAVIX_LOG_CHAIN, "Executing query for {} Metalink content", metalink_uri.getString());
197 if(tmp_err != NULL || (req.beginRequest(&tmp_err) <0) )
198 throw DavixException(davix_scope_meta(), tmp_err->getStatus(), tmp_err->getErrMsg());
199 if(httpcodeIsValid(req.getRequestCode()) == false){
200 throw DavixException(davix_scope_meta(), StatusCode::InvalidServerResponse, fmt::format("Unable to get Metalink file, error HTTP {}", req.getRequestCode()));
201 }
202
203 dav_ssize_t read_size;
204 do{
205 char buffer[2049];
206 buffer[2048] = '\0';
207 if( (read_size = req.readSegment(buffer, 2048, &tmp_err)) < 0)
208 throw DavixException(davix_scope_meta(), tmp_err->getStatus(), tmp_err->getErrMsg());
209 parser.parseChunk(buffer, read_size);
210 }while(read_size > 0);
211
212 req.endRequest(NULL);
213 return vec.size();
214 }
215
216 void davix_file_get_all_replicas_metalink( Context & c, const Uri & uri,
217 const RequestParams & _params, std::vector<DavFile> & vec){
218 Uri metalink;
219 if(davix_get_metalink_url(c, uri,
220 _params, metalink) > 0
221 && davix_file_get_metalink_to_vfile(c, metalink,_params, vec) > 0){
222 return;
223
224 }
225 throw DavixException(davix_scope_meta(), StatusCode::OperationNonSupported, "Server does not support Metalink standard");
226 }
227
228
229 MetalinkOps::MetalinkOps()
230 {
231
232 }
233
234 MetalinkOps::~MetalinkOps(){
235
236 }
237
238
239
240 std::vector<File> & MetalinkOps::getReplicas(IOChainContext & iocontext, std::vector<File> &vec){
241 davix_file_get_all_replicas_metalink(iocontext._context, iocontext._uri, iocontext._reqparams, vec);
242 return vec;
243 }
244
245 StatInfo & MetalinkOps::statInfo(IOChainContext &iocontext, StatInfo &st_info){
246 FuncStatInfo func(std::bind(&HttpIOChain::statInfo, _next.get(), std::placeholders::_1, std::ref(st_info)));
247 return metalinkExecutor<FuncStatInfo, StatInfo &>(*this, iocontext, func);
248 }
249
250 dav_ssize_t MetalinkOps::read(IOChainContext &iocontext, void *buf, dav_size_t count){
251 FuncIO func(std::bind(&HttpIOChain::read, _next.get(), std::placeholders::_1, buf, count));
252 return metalinkExecutor<FuncIO, dav_ssize_t>(*this, iocontext, func);
253 }
254
255 dav_ssize_t MetalinkOps::pread(IOChainContext &iocontext, void *buf, dav_size_t count, dav_off_t offset){
256
257 FuncIO func(std::bind(&HttpIOChain::pread, _next.get(), std::placeholders::_1, buf, count, offset));
258 return metalinkExecutor<FuncIO, dav_ssize_t>(*this, iocontext, func);
259 }
260
261
262 dav_ssize_t MetalinkOps::preadVec(IOChainContext & iocontext, const DavIOVecInput * input_vec,
263 DavIOVecOuput * output_vec,
264 const dav_size_t count_vec){
265
266 FuncIO func(std::bind(&HttpIOChain::preadVec, _next.get(), std::placeholders::_1, input_vec, output_vec, count_vec));
267 return metalinkExecutor<FuncIO, dav_ssize_t>(*this, iocontext, func);
268 }
269
270 // read to fd Metalink manager
271 dav_ssize_t MetalinkOps::readToFd(IOChainContext & iocontext, int fd, dav_size_t size){
272 FuncIO func(std::bind(&HttpIOChain::readToFd, _next.get(), std::placeholders::_1, fd, size));
273 return metalinkExecutor<FuncIO, dav_ssize_t>(*this, iocontext, func);
274 }
275
276
277
278
279
280
281
282
283
284
285
286 ///////////////////////////////////////////////////////////////////////
287 ///////////////////////////////////////////////////////////////////////
288
289 template<class Executor, class ReturnType>
290 ReturnType autoRetryExecutor(HttpIOChain & chain, IOChainContext & io_context, Executor fun){
291
292 (void) chain;
293 const int max_retry = io_context._reqparams->getOperationRetry();
294 const int retry_delay = io_context._reqparams->getOperationRetryDelay();
295 int retry =1;
296 const Uri & u = io_context._uri;
297
298 while(1){
299 io_context.checkTimeout();
300 try{
301 return fun(io_context);
302 }catch(DavixException & error){
303
304 // propagate fatal exceptions and connection error exceptions
305 propagateNonRecoverableExceptions(error);
306 // we can not recover from connexion timeout
307 // throw exception, cancel IO chain request
308 if(error.code() == StatusCode::ConnectionTimeout){
309 throw error;
310 }
311 // we should also just give up if the server responds with 403 or 405
312 else if(error.code() == StatusCode::PermissionRefused){
313 throw error;
314 }
315
316 DAVIX_SLOG(DAVIX_LOG_VERBOSE, DAVIX_LOG_CHAIN, "Negative result for operation: {}. After {} retry", error.what(), retry);
317 if( retry >= max_retry){
318 throw DavixException(error.scope(), error.code(), fmt::format("Result {} after {} attempts", error.what(), retry));
319 }
320 }catch(...){
321 DAVIX_SLOG(DAVIX_LOG_VERBOSE, DAVIX_LOG_CHAIN, "Operation failure: Unknown Error");
322 throw DavixException(davix_scope_io_buff(), StatusCode::UnknownError, fmt::format("Unrecoverable error from IOChain on {}", u));
323 }
324 ++retry;
325 sleep(retry_delay);
326 }
327 }
328
329
330 AutoRetryOps::AutoRetryOps(){
331
332 }
333
334
335 AutoRetryOps::~AutoRetryOps(){
336
337 }
338
339
340 StatInfo & AutoRetryOps::statInfo(IOChainContext &iocontext, StatInfo &st_info){
341 FuncStatInfo func(std::bind(&HttpIOChain::statInfo, _next.get(), std::placeholders::_1, std::ref(st_info)));
342 return autoRetryExecutor<FuncStatInfo, StatInfo &>(*this, iocontext, func);
343 }
344
345 dav_ssize_t AutoRetryOps::read(IOChainContext &iocontext, void *buf, dav_size_t count){
346 FuncIO func(std::bind(&HttpIOChain::read, _next.get(), std::placeholders::_1, buf, count));
347 return autoRetryExecutor<FuncIO, dav_ssize_t>(*this, iocontext, func);
348 }
349
350 dav_ssize_t AutoRetryOps::pread(IOChainContext &iocontext, void *buf, dav_size_t count, dav_off_t offset){
351
352 FuncIO func(std::bind(&HttpIOChain::pread, _next.get(), std::placeholders::_1, buf, count, offset));
353 return autoRetryExecutor<FuncIO, dav_ssize_t>(*this, iocontext, func);
354 }
355
356
357 dav_ssize_t AutoRetryOps::preadVec(IOChainContext & iocontext, const DavIOVecInput * input_vec,
358 DavIOVecOuput * output_vec,
359 const dav_size_t count_vec){
360
361 FuncIO func(std::bind(&HttpIOChain::preadVec, _next.get(), std::placeholders::_1, input_vec, output_vec, count_vec));
362 return autoRetryExecutor<FuncIO, dav_ssize_t>(*this, iocontext, func);
363 }
364
365 // read to fd Metalink manager
366 dav_ssize_t AutoRetryOps::readToFd(IOChainContext & iocontext, int fd, dav_size_t size){
367 FuncIO func(std::bind(&HttpIOChain::readToFd, _next.get(), std::placeholders::_1, fd, size));
368 return autoRetryExecutor<FuncIO, dav_ssize_t>(*this, iocontext, func);
369 }
370
371
372 } // namespace Davix