Debian Codegraph @

src/fileops/S3IO.cpp

raw
1 /*
2 * This File is part of Davix, The IO library for HTTP based protocols
3 * Copyright (C) CERN 2017
4 * Author: Georgios Bitzes <[email protected]>
5 *
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2.1 of the License, or (at your option) any later version.
10 *
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, write to the Free Software
18 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19 *
20 */
21
22 #include "S3IO.hpp"
23 #include <core/ContentProvider.hpp>
24 #include <utils/davix_logger_internal.hpp>
25 #include <xml/S3MultiPartInitiationParser.hpp>
26
27 #define SSTR(message) static_cast<std::ostringstream&>(std::ostringstream().flush() << message).str()
28
29 namespace Davix{
30
31 static bool is_s3_operation(IOChainContext & context){
32 if(context._reqparams->getProtocol() == RequestProtocol::AwsS3) {
33 return true;
34 }
35
36 return false;
37 }
38
39 static bool should_use_s3_multipart(IOChainContext & context, dav_size_t size) {
40 bool is_s3 = is_s3_operation(context);
41
42 if(!is_s3) return false;
43
44 if(context._uri.fragmentParamExists("forceMultiPart")) {
45
46 return true;
47 }
48
49 return size > (1024 * 1024 * 512); // 512 MB
50 }
51
52 S3IO::S3IO() {
53
54 }
55
56 S3IO::~S3IO() {
57
58 }
59
60 std::string S3IO::initiateMultipart(IOChainContext & iocontext) {
61 Uri url(iocontext._uri);
62 url.addQueryParam("uploads", "");
63
64 return initiateMultipart(iocontext, url);
65 }
66
67 std::string S3IO::initiateMultipart(IOChainContext & iocontext, const Uri &url) {
68 DavixError * tmp_err=NULL;
69
70 PostRequest req(iocontext._context, url, &tmp_err);
71 checkDavixError(&tmp_err);
72
73 req.setParameters(iocontext._reqparams);
74 req.setRequestBody("");
75 req.executeRequest(&tmp_err);
76 if(!tmp_err && httpcodeIsValid(req.getRequestCode()) == false){
77 httpcodeToDavixError(req.getRequestCode(), davix_scope_io_buff(),
78 "write error: ", &tmp_err);
79 }
80 checkDavixError(&tmp_err);
81
82 std::string response = req.getAnswerContent();
83 S3MultiPartInitiationParser parser;
84 if(parser.parseChunk(response) != 0) {
85 DavixError::setupError(&tmp_err, "S3::MultiPart", StatusCode::InvalidServerResponse, "Unable to parse server response for multi-part initiation");
86 }
87 checkDavixError(&tmp_err);
88
89
90 DAVIX_SLOG(DAVIX_LOG_DEBUG, DAVIX_LOG_CHAIN,
91 "S3IO: Obtained multi-part upload id {} for {}",
92 parser.getUploadId(), iocontext._uri);
93 return parser.getUploadId();
94 }
95
96 std::string S3IO::writeChunk(IOChainContext & iocontext, const char* buff, dav_size_t size, const std::string &uploadId, int partNumber) {
97 Uri url(iocontext._uri);
98 url.addQueryParam("uploadId", uploadId);
99 url.addQueryParam("partNumber", SSTR(partNumber));
100
101 return writeChunk(iocontext, buff, size, url, partNumber);
102 }
103
104 std::string S3IO::writeChunk(IOChainContext & iocontext, const char* buff, dav_size_t size, const Uri &url, int partNumber) {
105 DAVIX_SLOG(DAVIX_LOG_DEBUG, DAVIX_LOG_CHAIN, "writing chunk #{} with size {}", partNumber, size);
106
107 DavixError * tmp_err=NULL;
108 PutRequest req(iocontext._context, url, &tmp_err);
109 checkDavixError(&tmp_err);
110
111 req.setParameters(iocontext._reqparams);
112 req.setRequestBody(buff, size);
113 req.executeRequest(&tmp_err);
114 if(!tmp_err && httpcodeIsValid(req.getRequestCode()) == false){
115 httpcodeToDavixError(req.getRequestCode(), davix_scope_io_buff(),
116 "write error: ", &tmp_err);
117 }
118 DAVIX_SLOG(DAVIX_LOG_DEBUG, DAVIX_LOG_CHAIN, "write result size {}", size);
119 checkDavixError(&tmp_err);
120
121 std::string etag;
122 if(!req.getAnswerHeader("Etag", etag)) {
123 DavixError::setupError(&tmp_err, "S3::MultiPart", StatusCode::InvalidServerResponse, "Unable to retrieve chunk Etag, necessary when committing chunks");
124 }
125
126 DAVIX_SLOG(DAVIX_LOG_DEBUG, DAVIX_LOG_CHAIN, "chunk #{} written successfully, etag: {}", partNumber, etag);
127 return etag;
128 }
129
130 bool S3IO::commitChunks(IOChainContext & iocontext, const std::string &uploadId, const std::vector<std::string> &etags) {
131 Uri url(iocontext._uri);
132 url.addQueryParam("uploadId", uploadId);
133
134 commitChunks(iocontext, url, etags);
135 return true;
136 }
137
138 void S3IO::commitChunks(IOChainContext & iocontext, const Uri &url, const std::vector<std::string> &etags) {
139 DAVIX_SLOG(DAVIX_LOG_DEBUG, DAVIX_LOG_CHAIN, "committing {} chunks", etags.size());
140
141 std::ostringstream payload;
142 payload << "<CompleteMultipartUpload>";
143 for(size_t i = 1; i <= etags.size(); i++) {
144 payload << "<Part>";
145 payload << "<PartNumber>" << i << "</PartNumber>";
146 payload << "<ETag>" << etags[i-1] << "</ETag>";
147 payload << "</Part>";
148 }
149 payload << "</CompleteMultipartUpload>";
150
151 DavixError * tmp_err=NULL;
152 PostRequest req(iocontext._context, url, &tmp_err);
153 req.setParameters(iocontext._reqparams);
154 req.setRequestBody(payload.str());
155 req.executeRequest(&tmp_err);
156
157 if(!tmp_err && httpcodeIsValid(req.getRequestCode()) == false){
158 httpcodeToDavixError(req.getRequestCode(), davix_scope_io_buff(),
159 "write error: ", &tmp_err);
160 }
161 checkDavixError(&tmp_err);
162 }
163
164 static dav_size_t fillBufferWithProviderData(std::vector<char> &buffer, const dav_size_t maxChunkSize, ContentProvider &provider) {
165 dav_size_t written = 0u;
166 dav_size_t remaining = maxChunkSize;
167
168 while(true) {
169 dav_ssize_t bytesRead = provider.pullBytes(buffer.data(), remaining);
170 if(bytesRead < 0) {
171 throw DavixException(davix_scope_io_buff(), StatusCode::InvalidFileHandle, fmt::format("Error when reading from callback: {}", bytesRead));
172 }
173
174 remaining -= bytesRead;
175 written += bytesRead;
176
177 if(bytesRead == 0) {
178 DAVIX_SLOG(DAVIX_LOG_DEBUG, DAVIX_LOG_CHAIN, "Reached data provider EOF, received 0 bytes, even though asked for {}", remaining);
179 break; // EOF
180 }
181
182 if(remaining == 0) {
183 DAVIX_SLOG(DAVIX_LOG_DEBUG, DAVIX_LOG_CHAIN, "Data provider buffer has been filled");
184 break; // buffer is full
185 }
186 }
187
188 DAVIX_SLOG(DAVIX_LOG_DEBUG, DAVIX_LOG_CHAIN, "Retrieved {} bytes from data provider", written);
189 return written;
190 }
191
192 // write from a buffer
193 bool S3IO::writeFromBuffer(IOChainContext& iocontext, const char* buff,
194 dav_size_t size, const std::string& uploadId,
195 std::vector<std::string>& etags, int partNumber) {
196 etags.emplace_back(writeChunk(iocontext, buff, size, uploadId, partNumber));
197 return true;
198 }
199
200 // write from content provider
201 dav_ssize_t S3IO::writeFromProvider(IOChainContext & iocontext, ContentProvider &provider) {
202 if(!should_use_s3_multipart(iocontext, provider.getSize())) {
203 CHAIN_FORWARD(writeFromProvider(iocontext, provider));
204 }
205
206 DAVIX_SLOG(DAVIX_LOG_DEBUG, DAVIX_LOG_CHAIN, "Initiating multi-part upload towards {} to upload file with size {}", iocontext._uri, provider.getSize());
207 std::string uploadId = initiateMultipart(iocontext);
208
209 size_t remaining = provider.getSize();
210 const dav_size_t MAX_CHUNK_SIZE = 1024 * 1024 * 256; // 256 MB
211
212 std::vector<char> buffer;
213 buffer.resize(std::min(MAX_CHUNK_SIZE, (dav_size_t) provider.getSize()) + 10);
214
215 std::vector<std::string> etags;
216
217 size_t partNumber = 0;
218 while(remaining > 0) {
219 size_t toRead = std::min( (dav_size_t) provider.getSize(), MAX_CHUNK_SIZE);
220 DAVIX_SLOG(DAVIX_LOG_DEBUG, DAVIX_LOG_CHAIN, "S3IO write: toRead from cb {}", toRead);
221
222 dav_size_t bytesRead = fillBufferWithProviderData(buffer, MAX_CHUNK_SIZE, provider);
223 if(bytesRead == 0) break; // EOF
224
225 partNumber++;
226 etags.emplace_back(writeChunk(iocontext, buffer.data(), bytesRead, uploadId, partNumber));
227 }
228
229 commitChunks(iocontext, uploadId, etags);
230 return provider.getSize();
231 }
232
233 DynafedUris S3IO::retrieveDynafedUris(IOChainContext & iocontext, const std::string &uploadId, const std::string &pluginId, size_t nchunks) {
234 DynafedUris retval;
235
236 DavixError *tmp_err = NULL;
237 PutRequest req(iocontext._context, iocontext._uri, &tmp_err);
238 checkDavixError(&tmp_err);
239
240 req.setParameters(iocontext._reqparams);
241 req.addHeaderField("x-s3-uploadid", uploadId);
242 req.addHeaderField("x-ugrpluginid", pluginId);
243 req.addHeaderField("x-s3-upload-nchunks", SSTR(nchunks));
244 req.executeRequest(&tmp_err);
245
246 if(!tmp_err && httpcodeIsValid(req.getRequestCode()) == false){
247 httpcodeToDavixError(req.getRequestCode(), davix_scope_io_buff(),
248 "write error: ", &tmp_err);
249 }
250 checkDavixError(&tmp_err);
251
252 // We have a response, parse it into lines
253 retval.chunks = StrUtil::tokenSplit(req.getAnswerContent(), "\n");
254 if(!retval.chunks.empty()) {
255 retval.post = retval.chunks.back();
256 retval.chunks.pop_back();
257 }
258
259 DAVIX_SLOG(DAVIX_LOG_DEBUG, DAVIX_LOG_CHAIN, "retrieveDynafedUris: Obtained list with {} chunk URIs in total", retval.chunks.size());
260 return retval;
261 }
262
263 void S3IO::performUgrS3MultiPart(IOChainContext & iocontext, const std::string &posturl, const std::string &pluginId, ContentProvider &provider, DavixError **err) {
264 try {
265 Uri uri(posturl);
266 std::string uploadId = initiateMultipart(iocontext, posturl);
267
268 const dav_size_t MAX_CHUNK_SIZE = 1024 * 1024 * 256; // 256 MB
269 std::vector<char> buffer;
270 buffer.resize(std::min(MAX_CHUNK_SIZE, (dav_size_t) provider.getSize()) + 10);
271
272 size_t nchunks = (provider.getSize() / MAX_CHUNK_SIZE) + 2;
273 DynafedUris uris = retrieveDynafedUris(iocontext, uploadId, pluginId, nchunks);
274
275 if(uris.chunks.size() != nchunks) {
276 DAVIX_SLOG(DAVIX_LOG_WARNING, DAVIX_LOG_CHAIN, "Dynafed returned different number of URIs than expected: {} vs {]", "} retrieveDynafedUris: Obtained list with {} chunk URIs in total", uris.chunks.size(), nchunks);
277 throw DavixException("S3::MultiPart", StatusCode::InvalidServerResponse, "Dynafed returned different number of URIs than expected");
278 }
279
280 std::vector<std::string> etags;
281 size_t partNumber = 1;
282 uint64_t remaining = provider.getSize();
283
284 while(remaining > 0) {
285 dav_size_t bytesRetrieved = fillBufferWithProviderData(buffer, MAX_CHUNK_SIZE, provider);
286 if(bytesRetrieved == 0) {
287 break; // EOF
288 }
289
290 etags.emplace_back(writeChunk(iocontext, buffer.data(), bytesRetrieved, Uri(uris.chunks[partNumber-1]), partNumber));
291 partNumber++;
292 remaining -= bytesRetrieved;
293 }
294
295 commitChunks(iocontext, Uri(uris.post), etags);
296 }
297 CATCH_DAVIX(err);
298 }
299
300
301 }