Debian Codegraph @

src/fileops/iobuffmap.cpp

raw
1 /*
2 * This File is part of Davix, The IO library for HTTP based protocols
3 * Copyright (C) CERN 2013
4 * Author: Adrien Devresse <[email protected]>
5 *
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2.1 of the License, or (at your option) any later version.
10 *
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, write to the Free Software
18 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19 *
20 */
21
22 #include <davix_internal_config.hpp>
23 #include "iobuffmap.hpp"
24
25 #include <core/ContentProvider.hpp>
26 #include <utils/davix_types.hpp>
27 #include <request/httprequest.hpp>
28 #include <utils/davix_logger_internal.hpp>
29 #include <fileops/httpiovec.hpp>
30 #include <fileops/davmeta.hpp>
31 #include <system_utils/env_utils.hpp>
32
33
34 #include <sstream>
35 #include <string>
36
37 #include <cstring>
38 #include <cstdio>
39 #include <cstdlib>
40 #include <cmath>
41
42
43
44
45
46
47 namespace Davix {
48
49 dav_ssize_t read_truncated_segment_request(HttpRequest* req, void* buffer, dav_size_t size_read, dav_off_t off_set, DavixError**err);
50
51
52
53
54 dav_ssize_t read_segment_request(HttpRequest* req, void* buffer, dav_size_t size_read, DavixError**err){
55 DavixError* tmp_err=NULL;
56 dav_ssize_t ret, tmp_ret;
57 char* p_buff =(char*) buffer;
58 dav_size_t s_read= size_read;
59 ret = tmp_ret = 0;
60
61 DAVIX_SLOG(DAVIX_LOG_TRACE, DAVIX_LOG_CHAIN, "Davix::IOMap::readSegment: want to read {} bytes ", size_read);
62 do{
63 tmp_ret= req->readBlock(p_buff, s_read, &tmp_err);
64 if(tmp_ret > 0){ // tmp_ret bytes readed
65 ret += tmp_ret;
66 }
67 if(ret > 0 && ret < (dav_ssize_t) size_read){
68 p_buff+= tmp_ret;
69 s_read -= tmp_ret;
70 }
71 }while( tmp_ret > 0
72 && ret < (dav_ssize_t) size_read);
73
74 if(tmp_err){
75 DavixError::propagateError(err, tmp_err);
76 return -1;
77 }
78 DAVIX_SLOG(DAVIX_LOG_TRACE, DAVIX_LOG_CHAIN, "Davix::IOMap::readSegment: got {} bytes ", ret);
79 return ret;
80 }
81
82
83 dav_ssize_t read_truncated_segment_request(HttpRequest* req, void* buffer, dav_size_t size_read, dav_off_t off_set, DavixError**err){
84 DavixError* tmp_err=NULL;
85 dav_ssize_t ret=0, tmp_ret=0;
86 const dav_ssize_t begin_offset = (dav_ssize_t) off_set;
87 const dav_ssize_t ssize_read = size_read;
88 char * p_buffer = (char*) buffer;
89
90 while(ret < begin_offset && !tmp_err){
91 if( (ret + ssize_read) < begin_offset) // use buffer like trash for useless content
92 tmp_ret = req->readBlock(p_buffer, size_read, &tmp_err);
93 else
94 tmp_ret = req->readBlock(p_buffer, begin_offset - ret, &tmp_err);
95
96 if(tmp_ret == 0)
97 return 0;
98 ret += tmp_ret;
99 }
100
101 if(!tmp_err){
102 ret = read_segment_request(req, p_buffer, size_read, &tmp_err);
103 }
104
105 if(tmp_err){
106 DavixError::propagateError(err, tmp_err);
107 ret = -1;
108 }
109 return ret;
110 }
111
112 int get_valid_cache_file(FILE** stream, DavixError** err){
113 if(stream == NULL){
114 DavixError::setupError(err, davix_scope_io_buff(), StatusCode::InvalidFileHandle, "Invalid file stream");
115 return -1;
116 }
117 if( *stream == NULL){
118 if( ( *stream = tmpfile() ) == NULL){
119 std::ostringstream ss;
120 ss << "Error while file-cache creation: " << strerror(errno) << std::endl;
121 DavixError::setupError(err, davix_scope_io_buff(), StatusCode::SystemError, ss.str().c_str());
122 return -1;
123 }
124
125 }
126 return 0;
127 }
128
129 ///////////////////////
130 ///////////////////////
131 ///////////////////////
132
133 HttpIO::HttpIO()
134 {
135
136 }
137
138 HttpIO::~HttpIO(){
139 }
140
141
142 // read to dynamically allocated buffer
143 dav_ssize_t HttpIO::readFull(IOChainContext & iocontext, std::vector<char> & buffer){
144 DavixError * tmp_err=NULL;
145 dav_ssize_t ret = -1, total=0;
146
147 DAVIX_SCOPE_TRACE(DAVIX_LOG_CHAIN, fun_readFull);
148
149 GetRequest req (iocontext._context, iocontext._uri, &tmp_err);
150 if(!tmp_err){
151 RequestParams params(iocontext._reqparams);
152 req.setParameters(params);
153 ret = req.beginRequest(&tmp_err);
154 if(!tmp_err){
155 const dav_size_t s_chunk = (req.getAnswerSize() > 0)?(req.getAnswerSize()):DAVIX_BLOCK_SIZE;
156 buffer.reserve(buffer.size()+ s_chunk);
157
158 while ( (ret= req.readBlock( buffer, s_chunk, &tmp_err)) > 0){
159 total += (dav_size_t) ret;
160 }
161 if(!tmp_err && httpcodeIsValid(req.getRequestCode()) == false){
162 httpcodeToDavixError(req.getRequestCode(),davix_scope_io_buff(),"read error: ", &tmp_err);
163 ret = -1;
164 }
165 }
166 }
167
168 checkDavixError(&tmp_err);
169 return (ret>=0)?total:-1;
170 }
171
172
173 dav_ssize_t HttpIO::pread(IOChainContext & iocontext, void *buf, dav_size_t count, dav_off_t offset){
174 DavixError * tmp_err=NULL;
175 dav_ssize_t ret = -1;
176 DAVIX_SLOG(DAVIX_LOG_DEBUG, DAVIX_LOG_CHAIN, "pread operation for {} with size {} and offset {}", iocontext._uri, count, offset);
177 if(count ==0)
178 return 0;
179
180 HttpRequest req(iocontext._context, iocontext._uri, &tmp_err);
181
182 // Check Partial Content support via response header
183 // (EOS supports Partial Content but returns 200 instead of 206)
184 auto _supports_partial_content = [&req](dav_off_t offset, dav_size_t count) -> bool {
185 std::string header;
186 std::ostringstream oss;
187 oss << "bytes " << offset << "-" << ((count > 0) ? std::to_string(offset + count - 1) : "");
188 req.getAnswerHeader("Content-Range", header);
189 return (header.find(oss.str()) != std::string::npos);
190 };
191
192 if(tmp_err == NULL){
193 RequestParams params(iocontext._reqparams);
194 req.setParameters(params);
195 setup_offset_request(&req, &offset, &count,1);
196 if(req.beginRequest(&tmp_err) ==0){
197 if(req.getRequestCode() == 416 ){ // out of file, end of file
198 ret = 0; // end of file
199 DavixError::clearError(&tmp_err);
200 }else{
201 // partial request supported, just read !
202 if (req.getRequestCode() == 206 ||
203 (req.getRequestCode() == 200 && _supports_partial_content(offset, count))) {
204 ret = read_segment_request(&req, buf, count, &tmp_err);
205
206 // clean remaining content
207 if(!tmp_err) {
208 char buffer[255];
209 while( req.readBlock(buffer, 255, NULL) > 0);
210 }
211
212 }else if( req.getRequestCode() == 200){ // full request content -> skip useless content
213 ret = read_truncated_segment_request(&req, buf, count, offset, &tmp_err);
214 }else{
215 httpcodeToDavixError(req.getRequestCode(), davix_scope_http_request(),", while readding", &tmp_err);
216 }
217 }
218 }
219 req.endRequest(NULL);
220 }
221 DAVIX_SLOG(DAVIX_LOG_DEBUG, DAVIX_LOG_CHAIN, "end pread operation for {} ",iocontext._uri);
222 checkDavixError(&tmp_err);
223 return ret;
224 }
225
226 #define SSTR(message) static_cast<std::ostringstream&>(std::ostringstream().flush() << message).str()
227
228 dav_ssize_t HttpIO::readToFd(IOChainContext & iocontext, int fd, dav_size_t read_size){
229 DavixError * tmp_err=NULL;
230 dav_ssize_t ret = -1;
231
232 if(iocontext.fdHandler.fd != fd) {
233 iocontext.fdHandler.fd = fd;
234 iocontext.fdHandler.bytes_written_to_fd = 0;
235 }
236
237 DAVIX_SCOPE_TRACE(DAVIX_LOG_CHAIN, fun_readToFd);
238 DAVIX_SLOG(DAVIX_LOG_DEBUG, DAVIX_LOG_CHAIN, "request size {}", read_size);
239 GetRequest req (iocontext._context, iocontext._uri, &tmp_err);
240 if(!tmp_err){
241 RequestParams params(iocontext._reqparams);
242 req.setParameters(iocontext._reqparams);
243 if(iocontext.fdHandler.bytes_written_to_fd > 0) {
244 DAVIX_SLOG(DAVIX_LOG_WARNING, DAVIX_LOG_CHAIN, "{} bytes were already written to fd before transfer failed; attempting to resume from that point on", iocontext.fdHandler.bytes_written_to_fd);
245 req.addHeaderField("Range", SSTR("bytes=" << iocontext.fdHandler.bytes_written_to_fd << "-"));
246 }
247
248 ret = req.beginRequest(&tmp_err);
249 if(!tmp_err){
250 if(httpcodeIsValid(req.getRequestCode()) == false){
251 httpcodeToDavixError(req.getRequestCode(),davix_scope_io_buff(),"read error: ", &tmp_err);
252 ret = -1;
253 }else{
254 ret= req.readToFd(fd, read_size, &tmp_err);
255 }
256 }
257 }
258
259 if(ret > 0) {
260 iocontext.fdHandler.bytes_written_to_fd += ret;
261 }
262
263 DAVIX_SLOG(DAVIX_LOG_DEBUG, DAVIX_LOG_CHAIN, "read size {}", ret);
264 checkDavixError(&tmp_err);
265 return ret;
266 }
267
268 dav_ssize_t HttpIO::writeFromProvider(IOChainContext & iocontext, ContentProvider &provider) {
269 DavixError * tmp_err=NULL;
270
271 DAVIX_SLOG(DAVIX_LOG_DEBUG, DAVIX_LOG_CHAIN, "write size {}", provider.getSize());
272 PutRequest req (iocontext._context,iocontext._uri, &tmp_err);
273 if(!tmp_err){
274 RequestParams params(iocontext._reqparams);
275 req.setParameters(params);
276 req.setRequestBody(provider);
277 req.executeRequest(&tmp_err);
278 if(!tmp_err && httpcodeIsValid(req.getRequestCode()) == false){
279 httpcodeToDavixError(req.getRequestCode(), davix_scope_io_buff(),
280 "write error: ", &tmp_err);
281 }
282 }
283
284 DAVIX_SLOG(DAVIX_LOG_DEBUG, DAVIX_LOG_CHAIN, "write result size {}", provider.getSize());
285 checkDavixError(&tmp_err);
286 return provider.getSize();
287 }
288
289
290 /////////////////////////////////////////////////////
291 /////////////////////////////////////////////////////
292 ///
293
294
295 struct IOBufferLocalFile{
296 IOBufferLocalFile(int fd, const std::string & filepath): _fd(fd), _filepath(filepath){}
297 virtual ~IOBufferLocalFile(){
298 DAVIX_SLOG(DAVIX_LOG_TRACE, DAVIX_LOG_CHAIN, "Delete tmp file {}", _filepath);
299 unlink(_filepath.c_str());
300 close(_fd);
301 }
302
303 int _fd;
304 std::string _filepath;
305 };
306
307 HttpIOBuffer::HttpIOBuffer() :
308 HttpIOChain(),
309 _file_size(0),
310 _file_exist(false),
311 _pos(false),
312 _opened(false),
313 _last_advise(AdviseAuto),
314 _rwlock(),
315 _read_pos(0),
316 _read_endfile(false),
317 _read_req(NULL)
318 {
319
320 }
321
322 HttpIOBuffer::~HttpIOBuffer(){
323 delete _read_req;
324 }
325
326 IOBufferLocalFile* createLocalBuffer(){
327 std::string staging = EnvUtils::getEnv("DAVIX_STAGING_AREA", "/tmp");
328 staging += "/.davix_tmp_file_XXXXXX";
329
330 char buffer[1024];
331 strncpy(buffer, staging.c_str(), 1023);
332
333 int fd;
334 if( (fd = mkstemp(buffer)) < 0){
335 DAVIX_SLOG(DAVIX_LOG_TRACE, DAVIX_LOG_CHAIN, "Error during temporary file creation for HTTPIO {}: {}", buffer, strerror(errno));
336 return NULL;
337 }
338 return new IOBufferLocalFile(fd, buffer);
339 }
340
341 bool HttpIOBuffer::open(IOChainContext & iocontext, int flags){
342 bool res = false;
343 if(_opened)
344 return true;
345
346 struct StatInfo infos;
347
348 try{
349 _start->statInfo(iocontext, infos);
350
351 if( (flags & O_EXCL) && ( flags & O_CREAT)){
352 throw DavixException(davix_scope_io_buff(),
353 StatusCode::FileExist, "file exist and O_EXCL flag usedin open");
354 }else{
355 _file_size = infos.size;
356 _file_exist = true;
357 _opened = true;
358 }
359 }catch(DavixException & e){
360 if(e.code() == StatusCode::FileNotFound
361 && (flags & O_CREAT)
362 && ((flags & O_RDWR) || (flags & O_WRONLY))){
363 _file_size = 0;
364 _file_exist = false;
365 _opened = true;
366 _local.reset(createLocalBuffer());
367 } else if (e.code() == StatusCode::PermissionRefused
368 && (flags == O_RDONLY)) {
369 std::string errmsg = e.what();
370
371 // Some servers don't support HEAD request
372 // When HTTP 405 is returned, move on to GET request
373 if (errmsg.rfind("HTTP 405", 0) == 0) {
374 _file_size = 0;
375 _file_exist = true;
376 _opened = true;
377 }
378 }
379
380 if (!_opened) {
381 throw e;
382 }
383 }
384
385 DAVIX_SLOG(DAVIX_LOG_TRACE, DAVIX_LOG_CHAIN, "File open {}, size: {}", iocontext._uri, _file_size);
386 return res;
387 }
388
389 dav_ssize_t HttpIOBuffer::read(IOChainContext & iocontext, void *buf, dav_size_t count){
390 std::lock_guard<std::recursive_mutex> l(_rwlock);
391 DavixError* tmp_err = NULL;
392 dav_ssize_t ret =-1;
393
394 if(_pos ==0) // reset read ahead offset to default if try to read a full file
395 resetIO(iocontext);
396 if(_pos == _read_pos && isAdviseFullRead()){
397 // try read ahead strategie
398 ret = readInternal(iocontext, buf, count);
399 }else{ // fallback on partial read
400 ret = _start->pread(iocontext, buf, count, _pos);
401 }
402 if(ret > 0)
403 _pos += ret;
404
405 checkDavixError(&tmp_err);
406 return ret;
407 }
408
409
410 dav_ssize_t HttpIOBuffer::readInternal(IOChainContext & iocontext, void *buffer, dav_size_t size_read){
411 dav_ssize_t ret = -1;
412 DavixError * tmp_err=NULL;
413
414
415 if(_read_endfile)
416 return 0;
417
418 if( _read_req == NULL
419 && (_read_req = new HttpRequest(iocontext._context, iocontext._uri, &tmp_err)) != NULL
420 && tmp_err == NULL ){
421 RequestParams params(iocontext._reqparams);
422 _read_req->setParameters(params);
423 if(_read_req->beginRequest(&tmp_err) ==0
424 && (_read_req->getRequestCode() != 200)){
425 httpcodeToDavixError(_read_req->getRequestCode(),davix_scope_http_request(),", while readding", &tmp_err);
426 delete _read_req;
427 _read_req = NULL;
428
429 }
430 if(tmp_err){
431 delete _read_req;
432 _read_req = NULL;
433 ret = -1;
434 }
435
436 }
437
438 if(_read_req != NULL){ // valid request -> proceed to read
439 ret = read_segment_request(_read_req, buffer, size_read, &tmp_err);
440 if(ret > 0){
441 _read_pos += ret;
442 if(ret < (dav_ssize_t) size_read){ // end of file
443 _read_endfile =true;
444 _read_req->endRequest(NULL);
445 }
446 }
447 }
448
449 if((_read_endfile || ret < 0) && _read_req){
450 delete _read_req;
451 _read_req = NULL;
452 }
453
454 checkDavixError(&tmp_err);
455 return ret;
456 }
457
458
459
460 void HttpIOBuffer::prefetchInfo(IOChainContext & iocontext, off_t offset, dav_size_t size_read, advise_t adv){
461 (void) iocontext;
462 (void) offset;
463 (void) size_read;
464 _last_advise = adv;
465 }
466
467
468 void HttpIOBuffer::resetIO(IOChainContext & iocontext){
469 std::lock_guard<std::recursive_mutex> l(_rwlock);
470
471 if(_read_req){
472 delete _read_req;
473 _read_req = NULL;
474 }
475 _read_pos =0;
476 commitLocal(iocontext);
477 }
478
479
480 void HttpIOBuffer::commitLocal(IOChainContext & iocontext){
481 std::lock_guard<std::recursive_mutex> l(_rwlock);
482 if(_local.get()){
483 struct stat st;
484 memset(&st,0, sizeof(struct stat));
485 fstat(_local->_fd, &st);
486 DAVIX_SLOG(DAVIX_LOG_TRACE, DAVIX_LOG_CHAIN, "Commit local file modifications, {} bytes", st.st_size);
487
488 FdContentProvider provider(_local->_fd, 0, st.st_size);
489 _start->writeFromProvider(iocontext, provider);
490 _local.reset();
491 }
492 }
493
494
495 dav_off_t HttpIOBuffer::lseek(IOChainContext & iocontext, dav_off_t offset, int flags){
496 (void) iocontext;
497 std::lock_guard<std::recursive_mutex> l(_rwlock);
498 switch(flags){
499 case SEEK_CUR:
500 _pos += offset;
501 break;
502 case SEEK_END:
503 _pos = ( _file_size += offset);
504 break;
505 case SEEK_SET:
506 default:
507 _pos = offset;
508 break;
509
510 }
511 return _pos;
512 }
513
514
515 dav_ssize_t HttpIOBuffer::write(IOChainContext & iocontext, const void *buf, dav_size_t count){
516 (void) iocontext;
517 std::lock_guard<std::recursive_mutex> l(_rwlock);
518 dav_ssize_t ret =-1;
519 dav_size_t write_len = count;
520
521 if(!_opened) {
522 throw DavixException(davix_scope_io_buff(), StatusCode::SystemError, "Impossible to write, descriptor has not been opened");
523 }
524
525 if(_local.get() == NULL) {
526 throw DavixException(davix_scope_io_buff(), StatusCode::SystemError, "Impossible to write, no buffer. (file was opened only for reading?)");
527 }
528
529 do{
530 ret = pwrite(_local->_fd, buf, static_cast<size_t>(count), _pos);
531
532 if (ret == -1 && errno == EINTR) {
533 continue;
534 } else if (ret < 0) {
535 throw DavixException(davix_scope_io_buff(),
536 StatusCode::SystemError, std::string("Impossible to write to fd").append(strerror(errno)));
537 } else {
538 _pos += ret;
539 write_len -= ret;
540 }
541 }while(write_len >0);
542
543 return (count - write_len);
544 }
545
546
547 } // namespace Davix