Debian Codegraph @

src/fileops/AzureIO.cpp

raw
1 /*
2 * This File is part of Davix, The IO library for HTTP based protocols
3 * Copyright (C) CERN 2017
4 * Author: Georgios Bitzes <[email protected]>
5 *
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2.1 of the License, or (at your option) any later version.
10 *
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, write to the Free Software
18 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19 *
20 */
21
22 #include "AzureIO.hpp"
23 #include <utils/davix_logger_internal.hpp>
24 #include <core/ContentProvider.hpp>
25
26 #include <iomanip>
27 #include <uuid/uuid.h>
28 #include <algorithm>
29 #define SSTR(message) static_cast<std::ostringstream&>(std::ostringstream().flush() << message).str()
30
31 namespace Davix{
32
33 AzureIO::AzureIO() : HttpIOChain() { }
34
35 AzureIO::~AzureIO() {
36
37 }
38
39 static bool is_azure_operation(IOChainContext & context){
40 if(context._reqparams->getProtocol() == RequestProtocol::Azure) {
41 return true;
42 }
43
44 if(context._uri.queryParamExists("sig") &&
45 context._uri.queryParamExists("sr") &&
46 context._uri.queryParamExists("sp")) {
47
48 return true;
49 }
50
51 return false;
52 }
53
54 static std::string stringifyBlockID(const std::string &prefix, size_t blockid) {
55 std::string strblockid = SSTR(prefix << "+" << std::setfill('0') << std::setw(10) << blockid); // TODO ensure fixed size
56 return Base64::base64_encode( (unsigned char*) strblockid.c_str(), strblockid.size());
57 }
58
59 void AzureIO::writeChunk(IOChainContext & iocontext, const char* buff, dav_size_t size, const std::string &blockid) {
60 DavixError * tmp_err=NULL;
61 Uri url(iocontext._uri);
62 url.addQueryParam("comp", "block");
63 url.addQueryParam("blockid", blockid);
64 url.addFragmentParam("azuremechanism", "true");
65
66 PutRequest req(iocontext._context, url, &tmp_err);
67 if(!tmp_err){
68 RequestParams params(iocontext._reqparams);
69 params.addHeader("x-ms-blob-type", "BlockBlob");
70 req.setParameters(params);
71 req.setRequestBody(buff, size);
72 req.executeRequest(&tmp_err);
73 if(!tmp_err && httpcodeIsValid(req.getRequestCode()) == false){
74 httpcodeToDavixError(req.getRequestCode(), davix_scope_io_buff(),
75 "write error: ", &tmp_err);
76 }
77 }
78 DAVIX_SLOG(DAVIX_LOG_DEBUG, DAVIX_LOG_CHAIN, "write result size {}", size);
79 checkDavixError(&tmp_err);
80 }
81
82 void AzureIO::commitChunks(IOChainContext & iocontext, const std::vector<std::string> &blocklist) {
83 DAVIX_SLOG(DAVIX_LOG_DEBUG, DAVIX_LOG_CHAIN, "Azure write: committing {} blocks", blocklist.size());
84
85 DavixError * tmp_err=NULL;
86 Uri url(iocontext._uri);
87 url.addQueryParam("comp", "blocklist");
88 url.addFragmentParam("azuremechanism", "true");
89
90 std::stringstream body;
91 body << "<?xml version=\"1.0\" encoding=\"utf-8\"?><BlockList>";
92 for(size_t i = 0; i < blocklist.size(); i++) {
93 body << "<Latest>" << blocklist[i] << "</Latest>";
94 }
95 body << "</BlockList>";
96
97 PutRequest req(iocontext._context, url, &tmp_err);
98 if(!tmp_err){
99 RequestParams params(iocontext._reqparams);
100 req.setParameters(params);
101 req.setRequestBody(body.str());
102 req.executeRequest(&tmp_err);
103 if(!tmp_err && httpcodeIsValid(req.getRequestCode()) == false){
104 httpcodeToDavixError(req.getRequestCode(), davix_scope_io_buff(),
105 "write error: ", &tmp_err);
106 }
107 }
108 checkDavixError(&tmp_err);
109 }
110
111 static std::string uuid_to_string(uuid_t uuid) {
112 std::stringstream ss;
113 for(size_t i = 0; i < 16; i++) {
114 ss << std::setw(2) << std::setfill('0') << std::hex << (unsigned int) uuid[i];
115 }
116 return ss.str();
117 }
118
119 static std::string get_uuid() {
120 uuid_t uuid;
121 uuid_generate_random(uuid);
122 return uuid_to_string(uuid);
123 }
124
125 // write from content provider
126 dav_ssize_t AzureIO::writeFromProvider(IOChainContext & iocontext, ContentProvider &provider) {
127 if(!is_azure_operation(iocontext)) {
128 CHAIN_FORWARD(writeFromProvider(iocontext, provider));
129 }
130
131 std::vector<std::string> blockIDs;
132
133 DAVIX_SLOG(DAVIX_LOG_DEBUG, DAVIX_LOG_CHAIN, "Azure write: size {}, splitting into blocks", provider.getSize());
134 std::vector<char> buffer;
135
136 const dav_size_t MAX_CHUNK_SIZE = 1024 * 1024 * 100; // 100 MB
137 buffer.resize(std::min(MAX_CHUNK_SIZE, (dav_size_t) provider.getSize()) + 10);
138
139 // generate UUID to use as blockid prefix
140 std::string prefix = get_uuid();
141
142 size_t blockid = 0;
143 size_t remaining = provider.getSize();
144 while(remaining > 0) {
145 size_t toRead = std::min( (dav_size_t) provider.getSize(), MAX_CHUNK_SIZE);
146 DAVIX_SLOG(DAVIX_LOG_DEBUG, DAVIX_LOG_CHAIN, "Azure write: toRead from cb {}", toRead);
147
148 size_t bytesRead = provider.pullBytes(buffer.data(), toRead);
149 DAVIX_SLOG(DAVIX_LOG_DEBUG, DAVIX_LOG_CHAIN, "Azure write: bytesRead from cb {}", bytesRead);
150 if(bytesRead == 0) break; // EOF?
151
152 blockIDs.push_back(stringifyBlockID(prefix, blockid));
153 writeChunk(iocontext, buffer.data(), bytesRead, blockIDs.back());
154 blockid++;
155
156 remaining -= bytesRead;
157 DAVIX_SLOG(DAVIX_LOG_DEBUG, DAVIX_LOG_CHAIN, "Azure write: remaining bytes {}", remaining);
158 }
159
160 // Now let's commit the blobs
161 commitChunks(iocontext, blockIDs);
162 return provider.getSize();
163
164 }
165
166
167 }