Debian Codegraph @

src/utils/davix_s3_utils.cpp

raw
1 /*
2 * This File is part of Davix, The IO library for HTTP based protocols
3 * Copyright (C) CERN 2013
4 * Author: Adrien Devresse <[email protected]>
5 *
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2.1 of the License, or (at your option) any later version.
10 *
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, write to the Free Software
18 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19 *
20 */
21
22 #include <utils/davix_s3_utils.hpp>
23
24 #include <iomanip>
25 #include <ctime>
26 #include <cstring>
27 #include <davix_internal.hpp>
28 #include <utils/stringutils.hpp>
29 #include "libs/datetime/datetime_utils.hpp"
30 #include <utils/davix_logger_internal.hpp>
31 #include <utils/davix_utils_internal.hpp>
32 #include "libs/alibxx/crypto/base64.hpp"
33 #include "libs/alibxx/crypto/hmacsha.hpp"
34 #include <openssl/md5.h>
35 #include <sys/mman.h>
36
37 template <typename T>
38 static inline std::string SSTR(const T& v)
39 {
40 std::ostringstream stream;
41 stream << v;
42 return stream.str();
43 }
44
45
46 namespace Davix{
47
48
49 const std::string prefix_s3_header("x-amz-");
50 const std::string prefix_s3_date("x-amz-date");
51
52
53 std::string hexEncode(std::string input, std::string separator="") {
54 std::ostringstream ss;
55 for(std::string::iterator it = input.begin(); it != input.end(); it++) {
56 ss << std::setw(2) << std::setfill('0') << std::hex << (unsigned int) ( (unsigned char) *it) << separator;
57 }
58 return ss.str();
59 }
60
61 std::string getAwsReqToken(const std::string & stringToSign, const std::string & private_key){
62 std::ostringstream ss;
63 const std::string hmac = hmac_sha1(private_key, stringToSign);
64 ss << Base64::base64_encode((unsigned char*) hmac.c_str(), hmac.size());
65 return ss.str();
66 }
67
68 std::string getAwsAuthorizationField(const std::string & stringToSign, const std::string & private_key, const std::string & access_key){
69 std::ostringstream ss;
70 const std::string hmac = hmac_sha1(private_key, stringToSign);
71 ss << "AWS "<< access_key << ":" << Base64::base64_encode((unsigned char*) hmac.c_str(), hmac.size());
72 return ss.str();
73 }
74
75 std::string getAwsSignaturev4(const std::string & stringToSign, const std::string & private_key,
76 const std::string & region, const std::string & service) {
77 const std::string date = current_time("%Y%m%d");
78 const std::string kDate = hmac_sha256("AWS4" + private_key, date);
79 const std::string kRegion = hmac_sha256(kDate, region);
80 const std::string kService = hmac_sha256(kRegion, service);
81
82 const std::string c = "aws4_request";
83 const std::string kSigning = hmac_sha256(kService, c);
84 return hexEncode(hmac_sha256(kSigning, stringToSign));
85 }
86
87 namespace S3{
88 std::string extract_s3_provider(const Uri & uri) {
89 const std::string & hostname = uri.getHost();
90 std::string::const_iterator it = std::find(hostname.begin(), hostname.end(),'.');
91 return std::string(it, hostname.end());
92 }
93
94 std::string extract_s3_bucket(const Uri & uri, bool aws_alternate){
95 if(!aws_alternate) {
96 const std::string & hostname = uri.getHost();
97 std::string::const_iterator it = std::find(hostname.begin(), hostname.end(),'.');
98 return std::string(hostname.begin(), it);
99 }
100 else {
101 const std::string path = uri.getPath();
102 std::size_t pos = path.find("/", 1);
103 if(pos == std::string::npos) {
104 return path.substr(1, path.size()-1);
105 }
106 return path.substr(1, pos-1);
107 }
108 }
109
110 std::string extract_s3_path(const Uri & uri, bool aws_alternate) {
111 const std::string path = uri.getPath();
112 if(!aws_alternate) return path;
113
114 std::size_t pos = path.find("/", 1);
115 if(pos == std::string::npos) {
116 return std::string("/");
117 }
118
119 return path.substr(pos, path.size());
120 }
121
122 std::string detect_region(const Uri &uri) {
123 DavixError *err = NULL;
124 Context context;
125 GetRequest req(context, uri, &err);
126
127 if(err) {
128 return "";
129 }
130
131 RequestParams p;
132 p.setAwsRegion("null");
133 p.setOperationRetry(0);
134
135 req.setParameters(p);
136 req.executeRequest(&err);
137
138 DavixError::clearError(&err);
139
140 std::string region;
141 if(req.getAnswerHeader("x-amz-region", region)) {
142 return region;
143 }
144 return "";
145 }
146
147 static std::string get_md5(const HeaderVec & vec){
148 for(HeaderVec::const_iterator it = vec.begin(); it < vec.end(); it++){
149 if( StrUtil::compare_ncase(it->first, "Content-MD5") ==0){
150 return it->second;
151 }
152 }
153 return "";
154 }
155
156 static std::string get_type(const HeaderVec & vec){
157 for(HeaderVec::const_iterator it = vec.begin(); it < vec.end(); it++){
158 if( StrUtil::compare_ncase(it->first, "Content-Type") ==0){
159 return it->second;
160 }
161 }
162 return "";
163 }
164
165 static std::string get_date(HeaderVec & vec){
166 for(HeaderVec::iterator it = vec.begin(); it < vec.end(); it++){
167 if( StrUtil::compare_ncase(it->first, "date") ==0){
168 return it->second;
169 }
170 }
171
172 std::string date = current_time("%a, %d %b %Y %H:%M:%S %z");
173
174 // push date
175 vec.push_back(std::pair<std::string, std::string>("Date", date));
176 return date;
177 }
178
179
180 bool matchAmzheaders(const std::string & header_key){
181 return (StrUtil::compare_ncase(header_key, prefix_s3_header, prefix_s3_header.size()) == 0
182 && StrUtil::compare_ncase(header_key, prefix_s3_date) != 0);
183 }
184
185 HeaderVec getAmzCanonHeaders_vec(const HeaderVec & headers){
186 HeaderVec canon_amz_headers;
187
188 for(HeaderVec::const_iterator it = headers.begin(); it < headers.end(); ++it){
189 std::string header_key = (*it).first, header_value = (*it).second;
190 StrUtil::toLower(StrUtil::trim(header_key));
191 StrUtil::trim(header_value);
192
193 if( matchAmzheaders(header_key)){
194 canon_amz_headers.push_back(*it);
195 }
196 }
197 return canon_amz_headers;
198 }
199
200 std::string getAmzCanonHeaders(const HeaderVec & headers) {
201 std::string canon_amz_headers;
202
203 for(HeaderVec::const_iterator it = headers.begin(); it < headers.end(); ++it){
204 std::string header_key = (*it).first, header_value = (*it).second;
205 StrUtil::toLower(StrUtil::trim(header_key));
206 StrUtil::trim(header_value);
207
208 if( matchAmzheaders(header_key)){
209 canon_amz_headers.reserve(canon_amz_headers.size() + header_key.size() + header_value.size() +1);
210 canon_amz_headers += header_key;
211 canon_amz_headers += ":";
212 canon_amz_headers += header_value;
213 canon_amz_headers += "\n";
214 }
215 }
216 return canon_amz_headers;
217 }
218
219 void signRequestv2(const RequestParams & params, const std::string & method, const Uri & url, HeaderVec & headers){
220 if(params.getAwsToken().size() != 0) {
221 headers.push_back(HeaderLine("x-amz-security-token", params.getAwsToken()));
222 }
223
224 std::ostringstream ss;
225
226 // construct Request token
227 ss << method << "\n"
228 << get_md5(headers) << "\n"
229 << get_type(headers) << "\n"
230 << get_date(headers) << "\n";
231
232 ss << getAmzCanonHeaders(headers);
233
234 // when using a path-based url, the bucket name is part of the path
235 if(params.getAwsAlternate()) {
236 ss << url.getPath();
237 }
238 else {
239 ss << '/' << extract_s3_bucket(url) << url.getPath();
240 }
241
242 ss << extractCanonicalizedResourceQueryParams(url);
243
244 DAVIX_SLOG(DAVIX_LOG_TRACE, DAVIX_LOG_S3, "String to sign (aws-alternate={}): {}", params.getAwsAlternate(), StrUtil::stringReplace(ss.str(), "\n", "\\n"));
245 headers.push_back(std::pair<std::string, std::string>("Authorization", getAwsAuthorizationField(ss.str(), params.getAwsAutorizationKeys().first, params.getAwsAutorizationKeys().second)));
246 }
247
248
249 // Sign an S3 request by modifying the headers, not the URI
250 void signRequest(const RequestParams & params, const std::string & method, const Uri & url, HeaderVec & headers){
251 if(params.getAwsRegion().empty()) {
252 signRequestv2(params, method, url, headers);
253 }
254 else {
255 throw std::runtime_error("v4 header signing not yet implemented");
256 }
257 }
258
259 #define SSTR(message) static_cast<std::ostringstream&>(std::ostringstream().flush() << message).str()
260
261 Uri signURIv4(const RequestParams & params, const std::string & method, const Uri & url, const HeaderVec headers, const time_t expirationTime) {
262 // references
263 // http://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-header-based-auth.html
264 // http://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html
265 DAVIX_SLOG(DAVIX_LOG_VERBOSE, DAVIX_LOG_S3, "Using S3 v4 signature authentication");
266
267 // using a non-default port?
268 bool defaultPort;
269 std::string portPart;
270
271 defaultPort = (url.getPort() == 0);
272 defaultPort |= (url.getPort() == 80 && (url.getProtocol() == "http" || url.getProtocol() == "s3"));
273 defaultPort |= (url.getPort() == 443 && (url.getProtocol() == "https" || url.getProtocol() == "s3s"));
274
275 if(!defaultPort) portPart = SSTR(":" << url.getPort());
276
277 // calculate canonical headers
278 HeaderVec can_headers = getAmzCanonHeaders_vec(headers);
279 can_headers.push_back(HeaderLine("Host", SSTR(url.getHost() << portPart)));
280 std::sort(can_headers.begin(), can_headers.end());
281 std::ostringstream can_headers_str;
282 for(HeaderVec::iterator it = can_headers.begin(); it != can_headers.end(); it++) {
283 can_headers_str << StrUtil::toLower(it->first) << ":" << StrUtil::trim(it->second) << "\n";
284 }
285
286 std::ostringstream signed_headers;
287 for(HeaderVec::iterator it = can_headers.begin(); it != can_headers.end(); it++) {
288 signed_headers << StrUtil::toLower(it->first);
289 if(it+1 != can_headers.end()) {
290 signed_headers << ";";
291 }
292 }
293
294 // calculate query parameters
295 ParamVec query_params;
296 query_params.push_back(HeaderLine("X-Amz-Algorithm", "AWS4-HMAC-SHA256"));
297
298 std::ostringstream credential;
299 credential << params.getAwsAutorizationKeys().second
300 << "/" << current_time("%Y%m%d")
301 << "/" << params.getAwsRegion()
302 << "/" << "s3"
303 << "/" << "aws4_request";
304
305 query_params.push_back(HeaderLine("X-Amz-Credential", credential.str()));
306
307 if(params.getAwsToken().size() != 0) {
308 query_params.push_back(HeaderLine("X-Amz-Security-Token", params.getAwsToken()));
309 }
310
311 // calculate amz date
312 std::string amzdate = current_time("%Y%m%dT%H%M%SZ");
313 query_params.push_back(HeaderLine("X-Amz-Date", amzdate));
314
315 // add timeout
316 std::string expiration = SSTR(expirationTime);
317 query_params.push_back(HeaderLine("X-Amz-Expires", expiration));
318
319 // add amz signed headers
320 query_params.push_back(HeaderLine("X-Amz-SignedHeaders", signed_headers.str()));
321
322 // calculate *canonical* query string
323 ParamVec can_query_params;
324 for(ParamVec::iterator it = query_params.begin(); it != query_params.end(); it++) {
325 can_query_params.push_back(ParamLine(Uri::queryParamEscape(it->first), Uri::queryParamEscape(it->second)));
326 }
327
328 const ParamVec existing_params = url.getQueryVec();
329 for(ParamVec::const_iterator it = existing_params.begin(); it != existing_params.end(); it++) {
330 can_query_params.push_back(ParamLine(it->first, it->second));
331 }
332
333 std::sort(can_query_params.begin(), can_query_params.end());
334
335 std::ostringstream can_query_str;
336 for(ParamVec::iterator it = can_query_params.begin(); it != can_query_params.end(); it++) {
337 can_query_str << it->first << "=" << it->second;
338 if(it+1 != can_query_params.end()) {
339 can_query_str << "&";
340 }
341 }
342
343 // calculate canonical request
344 std::ostringstream canonical_request;
345 canonical_request << method << "\n"
346 << url.getPath() << "\n"
347 << can_query_str.str() << "\n"
348 << can_headers_str.str() << "\n"
349 << signed_headers.str() << "\n"
350 << "UNSIGNED-PAYLOAD";
351
352 DAVIX_SLOG(DAVIX_LOG_DEBUG, DAVIX_LOG_S3, "Canonical request: {}", canonical_request.str());
353 DAVIX_SLOG(DAVIX_LOG_DEBUG, DAVIX_LOG_S3, "Canonical request bytes: {}", hexEncode(canonical_request.str(), " "));
354
355 // calculate canonical request hash
356 std::string can_req_hash = sha256(canonical_request.str().c_str());
357 std::string encoded_hash = hexEncode(can_req_hash);
358 DAVIX_SLOG(DAVIX_LOG_DEBUG, DAVIX_LOG_S3, "Canonical request hash: {}", encoded_hash);
359
360 // calculate string to sign
361 std::ostringstream stringToSign;
362 stringToSign << "AWS4-HMAC-SHA256" << "\n"
363 << amzdate << "\n"
364 << current_time("%Y%m%d") << "/" << params.getAwsRegion() << "/s3/aws4_request" << "\n"
365 << encoded_hash;
366
367 DAVIX_SLOG(DAVIX_LOG_DEBUG, DAVIX_LOG_S3, "String to sign (aws-alternate={}): {}", params.getAwsAlternate(), StrUtil::stringReplace(stringToSign.str(), "\n", "\\n"));
368 DAVIX_SLOG(DAVIX_LOG_DEBUG, DAVIX_LOG_S3, "String to sign bytes: {}", hexEncode(stringToSign.str(), " "));
369
370 // whew.. now calculate the final signature
371 Uri signedUrl = url;
372
373 std::string signature = getAwsSignaturev4(stringToSign.str(), params.getAwsAutorizationKeys().first,
374 params.getAwsRegion(), "s3");
375
376 DAVIX_SLOG(DAVIX_LOG_VERBOSE, DAVIX_LOG_S3, "Signature: {}", signature);
377 signedUrl.addQueryParam("X-Amz-Signature", signature);
378
379 for(ParamVec::iterator it = query_params.begin(); it != query_params.end(); it++) {
380 signedUrl.addQueryParam(it->first, it->second);
381 }
382
383
384 DAVIX_SLOG(DAVIX_LOG_DEBUG, DAVIX_LOG_S3, "Original URL: {}", url);
385 DAVIX_SLOG(DAVIX_LOG_DEBUG, DAVIX_LOG_S3, "Signed URL: {}", signedUrl);
386 return signedUrl;
387 }
388
389 // return a signed s3 URI, does not modify the headers
390 Uri signURI(const RequestParams & params, const std::string & method, const Uri & url, HeaderVec headers, const time_t expirationTime) {
391 if(params.getAwsRegion().empty()) {
392 return tokenizeRequest(params, method, url, headers, std::time(0)+expirationTime);
393 }
394 else {
395 return signURIv4(params, method, url, headers, expirationTime);
396 }
397 }
398
399
400 Uri tokenizeRequest(const RequestParams & params, const std::string & method, const Uri & url, HeaderVec & headers, time_t expirationTime){
401 if(params.getAwsToken().size() != 0) {
402 headers.push_back(HeaderLine("x-amz-security-token", params.getAwsToken()));
403 }
404
405 std::ostringstream ss;
406
407 // construct Request token
408 ss << method << "\n"
409 << get_md5(headers) << "\n"
410 << get_type(headers) << "\n"
411 << static_cast<unsigned long long>(expirationTime) << "\n";
412
413 ss << getAmzCanonHeaders(headers);
414
415 // when using a path-based url, the bucket name is part of the path
416 if(params.getAwsAlternate()) {
417 ss << url.getPath();
418 }
419 else {
420 ss << '/' << extract_s3_bucket(url) << url.getPath();
421 }
422
423 ss << extractCanonicalizedResourceQueryParams(url);
424 const std::string signature = getAwsReqToken(ss.str(), params.getAwsAutorizationKeys().first);
425
426 Uri signedUri(url);
427
428 signedUri.addQueryParam("AWSAccessKeyId", params.getAwsAutorizationKeys().second);
429 signedUri.addQueryParam("Signature", signature);
430 signedUri.addQueryParam("Expires", SSTR(expirationTime));
431
432 // add amz headers as query parameters
433 for(HeaderVec::const_iterator it = headers.begin(); it < headers.end(); ++it){
434 if(matchAmzheaders(it->first)){
435 std::string key= it->first, value = it->second;
436 StrUtil::toLower(StrUtil::trim(key));
437 StrUtil::trim(value);
438
439 signedUri.addQueryParam(key, value);
440 }
441 }
442
443 return signedUri;
444 }
445
446
447 Uri s3UriTransformer(const Uri & original_url, const RequestParams & params, const bool addDelimiter){
448 std::string delimiter = "&delimiter=%2F";
449 std::string prefix = "?prefix=";
450 std::string maxKey = "&max-keys=";
451
452 std::string protocol;
453
454 const std::string url_string = original_url.getString();
455 std::size_t pos = url_string.find(':');
456
457 if(url_string.compare(pos-1,1,"s") == 0){
458 protocol = "s3s://";
459 }
460 else{
461 protocol = "s3://";
462 }
463
464 std::ostringstream ss;
465
466 ss << protocol << original_url.getHost();
467 if(original_url.getPort() > 0) {
468 ss << ":" << original_url.getPort();
469 }
470 ss << "/";
471
472 if(params.getAwsAlternate()) {
473 ss << extract_s3_bucket(original_url, params.getAwsAlternate()) << "/";
474 }
475
476 if(!original_url.getPath().empty()){ // there is something after '/', grab it
477 std::string tmp = extract_s3_path(original_url, params.getAwsAlternate());
478
479 // if prefix doesn't end with '/', add one to handle query on folder
480 if(tmp.compare(tmp.size()-1,1,"/") != 0)
481 tmp += "/";
482
483 tmp.erase(0,1);
484 prefix += Uri::queryParamEscape(tmp);
485 }
486
487 ss << prefix << maxKey << params.getS3MaxKey();
488
489 // skip delimiter if where we want to list everything after a certain prefix,
490 // useful in cases like GET Collection
491 if(addDelimiter)
492 ss << delimiter;
493
494 return Uri(ss.str());
495 }
496
497 time_t s3TimeConverter(std::string &s3time){
498 struct tm t;
499 memset(&t, 0, sizeof(struct tm));
500 size_t pos=0;
501 std::string tmp;
502
503 // check which datetime format is used
504 if(strptime(s3time.c_str(), "%a, %d %b %Y %H:%M:%S %z", &t) == NULL){
505 if((pos = s3time.find("T")) != std::string::npos){ // iso 8601
506 tmp = s3time.substr(0,pos) + " " + s3time.substr(pos+1,s3time.find('.',pos)-1);
507 strptime(tmp.c_str(), "%F %T", &t);
508 }
509 }
510 time_t mtime = timegm(&t);
511 return mtime;
512 }
513
514 // taken from dmlite
515 std::string hexPrinter(const unsigned char* data, dav_size_t nbytes){
516 char buffer[nbytes * 2 + 1];
517 char *p;
518
519 p = buffer;
520 for(dav_size_t offset = 0; offset < nbytes; ++offset, p+=2)
521 sprintf(p, "%02x", data[offset]);
522 *p = '\0';
523
524 return std::string(buffer);
525 }
526
527 int calculateMD5(std::string &input, std::string &output){
528 if(input.empty()) return -1;
529
530 unsigned char result_buf[MD5_DIGEST_LENGTH];
531 MD5((unsigned char*)input.c_str(), input.size(), result_buf);
532
533 output = Base64::base64_encode(result_buf, MD5_DIGEST_LENGTH);
534
535 return output.empty() ? -1 : 0;
536 }
537
538 int calculateMD5(int fd, std::string &output){
539 struct stat statbuf;
540 if(fstat(fd, &statbuf) < 0)
541 return -1;
542
543 unsigned char result_buf[MD5_DIGEST_LENGTH];
544
545 void* file_buf = mmap(0, statbuf.st_size, PROT_READ, MAP_SHARED, fd, 0);
546 MD5((unsigned char*)file_buf, statbuf.st_size, result_buf);
547 munmap(file_buf, statbuf.st_size);
548
549 output = Base64::base64_encode(result_buf, MD5_DIGEST_LENGTH);
550
551 return output.empty() ? -1 : 0;
552 }
553
554 } // S3
555
556
557 } // Davix