Debian Codegraph @

src/fileops/SwiftIO.cpp

raw
1 /*
2 * This File is part of Davix, The IO library for HTTP based protocols
3 * Copyright (C) CERN 2021
4 * Author: Shiting Long <[email protected]> (Forschungszentrum Juelich)
5 *
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2.1 of the License, or (at your option) any later version.
10 *
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, write to the Free Software
18 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19 *
20 */
21
22 #include "SwiftIO.hpp"
23 #include <core/ContentProvider.hpp>
24 #include <utils/davix_logger_internal.hpp>
25 #include <utils/davix_swift_utils.hpp>
26
27
28 namespace Davix{
29
30 static bool is_swift_operation(IOChainContext & context){
31 if(context._reqparams->getProtocol() == RequestProtocol::Swift) {
32 return true;
33 }
34
35 return false;
36 }
37
38 static bool should_use_swift_multipart(IOChainContext & context, dav_size_t size) {
39 bool is_swift = is_swift_operation(context);
40
41 if (!is_swift) return false;
42
43 if(context._uri.fragmentParamExists("forceMultiPart")) {
44
45 return true;
46 }
47
48 return size > (1024 * 1024 * 512); // 512 MB
49 }
50
51 SwiftIO::SwiftIO() {
52
53 }
54
55 SwiftIO::~SwiftIO() {
56
57 }
58
59 static dav_size_t fillBufferWithProviderData(std::vector<char> &buffer, const dav_size_t maxChunkSize, ContentProvider &provider) {
60 dav_size_t written = 0u;
61 dav_size_t remaining = maxChunkSize;
62
63 while(true) {
64 dav_ssize_t bytesRead = provider.pullBytes(buffer.data(), remaining);
65 if(bytesRead < 0) {
66 throw DavixException(davix_scope_io_buff(), StatusCode::InvalidFileHandle, fmt::format("Error when reading from callback: {}", bytesRead));
67 }
68
69 remaining -= bytesRead;
70 written += bytesRead;
71
72 if(bytesRead == 0) {
73 DAVIX_SLOG(DAVIX_LOG_DEBUG, DAVIX_LOG_CHAIN, "Reached data provider EOF, received 0 bytes, even though asked for {}", remaining);
74 break; // EOF
75 }
76
77 if(remaining == 0) {
78 DAVIX_SLOG(DAVIX_LOG_DEBUG, DAVIX_LOG_CHAIN, "Data provider buffer has been filled");
79 break; // buffer is full
80 }
81 }
82
83 DAVIX_SLOG(DAVIX_LOG_DEBUG, DAVIX_LOG_CHAIN, "Retrieved {} bytes from data provider", written);
84 return written;
85 }
86
87 std::string SwiftIO::writeChunk(IOChainContext &iocontext, const char *buff, dav_size_t size, int partNumber) {
88 Uri url(iocontext._uri);
89 url.setPath(url.getPath() + "/" + std::to_string(partNumber));
90
91 DAVIX_SLOG(DAVIX_LOG_DEBUG, DAVIX_LOG_CHAIN, "writing chunk #{} with size {}", partNumber, size);
92
93 DavixError * tmp_err=NULL;
94 PutRequest req(iocontext._context, url, &tmp_err);
95 checkDavixError(&tmp_err);
96
97 req.setParameters(iocontext._reqparams);
98 req.setRequestBody(buff, size);
99 req.executeRequest(&tmp_err);
100 if(!tmp_err && httpcodeIsValid(req.getRequestCode()) == false){
101 httpcodeToDavixError(req.getRequestCode(), davix_scope_io_buff(),
102 "write error: ", &tmp_err);
103 }
104 DAVIX_SLOG(DAVIX_LOG_DEBUG, DAVIX_LOG_CHAIN, "write result size {}", size);
105 checkDavixError(&tmp_err);
106
107 std::string etag;
108 if(!req.getAnswerHeader("Etag", etag)) {
109 DavixError::setupError(&tmp_err, "Swift::MultiPart", StatusCode::InvalidServerResponse, "Unable to retrieve chunk Etag, necessary when committing chunks");
110 }
111
112 DAVIX_SLOG(DAVIX_LOG_DEBUG, DAVIX_LOG_CHAIN, "chunk #{} written successfully, etag: {}", partNumber, etag);
113 return etag;
114 }
115
116 void SwiftIO::commitInlineChunks(IOChainContext & iocontext, const std::vector<Prop> &props, const size_t MaxManifestSegments){
117 Uri url(iocontext._uri);
118
119 size_t iterNum = 0;
120 size_t propBegin = 1;
121 size_t propEnd = MaxManifestSegments;
122
123 std::string lastEtag;
124 std::string path = url.getPath();
125
126 while(propBegin <= propEnd){
127 // generate a multipart manifest in json
128 std::ostringstream manifest;
129 manifest << "[";
130 if(iterNum > 0){
131 manifest << "{";
132 manifest << "\"path\":\"" << path << "-" << iterNum - 1 << "\",";
133 manifest << "\"etag\":" << lastEtag << "},";
134 }
135 for(size_t i = propBegin; i <= propEnd; i++) {
136 manifest << "{";
137 manifest << "\"path\":\"" << path << "/" << i << "\",";
138 manifest << "\"etag\":\"" << props[i - 1].first << "\",";
139 manifest << "\"size_bytes\":" << props[i - 1].second << "}";
140 if(i != propEnd) {
141 manifest << ',';
142 }
143 }
144 manifest << "]";
145
146 propBegin = propEnd + 1;
147 propEnd = std::min(propEnd + MaxManifestSegments - 1, props.size());
148
149 // upload the manifest
150 DavixError * tmp_err=NULL;
151 if(propBegin <= propEnd) {
152 url.setPath(path + "-" + std::to_string(iterNum));
153 } else{
154 url.setPath(path);
155 }
156 url.addQueryParam("multipart-manifest", "put");
157 PutRequest req(iocontext._context, url, &tmp_err);
158 req.addHeaderField("Content-Type", "application/json");
159 req.setParameters(iocontext._reqparams);
160 req.setRequestBody(manifest.str());
161 req.executeRequest(&tmp_err);
162
163 if(!tmp_err && httpcodeIsValid(req.getRequestCode()) == false){
164 httpcodeToDavixError(req.getRequestCode(), davix_scope_io_buff(),
165 "write error: ", &tmp_err);
166 }
167
168 if(!req.getAnswerHeader("Etag", lastEtag)) { // store the etag of the last manifest file
169 DavixError::setupError(&tmp_err, "Swift::MultiPart", StatusCode::InvalidServerResponse, "Unable to retrieve chunk Etag, necessary when committing chunks");
170 }
171 checkDavixError(&tmp_err);
172
173 manifest.clear();
174 iterNum++;
175 }
176 }
177
178 void SwiftIO::commitChunks(IOChainContext & iocontext, const std::vector<Prop> &props) {
179 Uri url(iocontext._uri);
180
181 DAVIX_SLOG(DAVIX_LOG_DEBUG, DAVIX_LOG_CHAIN, "committing {} chunks", props.size());
182
183 // generate a multipart manifest in json
184 std::ostringstream manifest;
185 manifest << "[";
186 for(size_t i = 1; i <= props.size(); i++) {
187 manifest << "{";
188 manifest << "\"path\":\"" << url.getPath() << "/" << i << "\",";
189 manifest << "\"etag\":\"" << props[i - 1].first << "\",";
190 manifest << "\"size_bytes\":" << props[i - 1].second << "}";
191 if(i != props.size()) {
192 manifest << ',';
193 }
194 }
195 manifest << "]";
196
197 // upload the manifest
198 DavixError * tmp_err=NULL;
199 url.addQueryParam("multipart-manifest", "put");
200 PutRequest req(iocontext._context, url, &tmp_err);
201 req.addHeaderField("Content-Type", "application/json");
202 req.setParameters(iocontext._reqparams);
203 req.setRequestBody(manifest.str());
204 req.executeRequest(&tmp_err);
205
206 if(!tmp_err && httpcodeIsValid(req.getRequestCode()) == false){
207 httpcodeToDavixError(req.getRequestCode(), davix_scope_io_buff(),
208 "write error: ", &tmp_err);
209 }
210 checkDavixError(&tmp_err);
211 }
212
213 dav_ssize_t SwiftIO::writeFromProvider(IOChainContext & iocontext, ContentProvider &provider) {
214 if(!should_use_swift_multipart(iocontext, provider.getSize())) {
215 CHAIN_FORWARD(writeFromProvider(iocontext, provider));
216 }
217
218 DAVIX_SLOG(DAVIX_LOG_DEBUG, DAVIX_LOG_CHAIN, "Initiating large file upload towards {} to upload file with size {}", iocontext._uri, provider.getSize());
219
220 size_t remaining = provider.getSize();
221 const dav_size_t MAX_CHUNK_SIZE = 1024 * 1024 * 256; // 256 MB
222 const size_t MAX_MANIFEST_SEGMENTS = 1000;
223
224 std::vector<char> buffer;
225 buffer.resize(std::min(MAX_CHUNK_SIZE, (dav_size_t) provider.getSize()) + 10);
226
227 std::vector<Prop> props;
228
229 size_t partNumber = 0;
230
231 while(remaining > 0) {
232 size_t toRead = std::min( (dav_size_t) provider.getSize(), MAX_CHUNK_SIZE);
233 DAVIX_SLOG(DAVIX_LOG_DEBUG, DAVIX_LOG_CHAIN, "SwiftIO write: toRead from cb {}", toRead);
234
235 dav_size_t bytesRead = fillBufferWithProviderData(buffer, MAX_CHUNK_SIZE, provider);
236 if(bytesRead == 0) break; // EOF
237
238 partNumber++;
239 props.emplace_back(writeChunk(iocontext, buffer.data(), bytesRead, partNumber), bytesRead);
240 }
241
242 if(props.size() > MAX_MANIFEST_SEGMENTS){ // if segment number is larger than max_manifest_segments (by default 1000), use inline segments
243 commitInlineChunks(iocontext, props, MAX_MANIFEST_SEGMENTS);
244 } else{
245 commitChunks(iocontext, props);
246 }
247 return provider.getSize();
248 }
249
250 }