| 1 |
/* |
| 2 |
* This File is part of Davix, The IO library for HTTP based protocols |
| 3 |
* Copyright (C) CERN 2018 |
| 4 |
* Authors: Georgios Bitzes <[email protected]> |
| 5 |
* Mario Lassnig <[email protected]> |
| 6 |
* |
| 7 |
* This library is free software; you can redistribute it and/or |
| 8 |
* modify it under the terms of the GNU Lesser General Public |
| 9 |
* License as published by the Free Software Foundation; either |
| 10 |
* version 2.1 of the License, or (at your option) any later version. |
| 11 |
* |
| 12 |
* This library is distributed in the hope that it will be useful, |
| 13 |
* but WITHOUT ANY WARRANTY; without even the implied warranty of |
| 14 |
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU |
| 15 |
* Lesser General Public License for more details. |
| 16 |
* |
| 17 |
* You should have received a copy of the GNU Lesser General Public |
| 18 |
* License along with this library; if not, write to the Free Software |
| 19 |
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA |
| 20 |
* |
| 21 |
*/ |
| 22 |
|
| 23 |
#include <fstream> |
| 24 |
#include <iomanip> |
| 25 |
#include <sstream> |
| 26 |
#include <rapidjson/document.h> |
| 27 |
#include <libs/alibxx/crypto/hmacsha.hpp> |
| 28 |
#include <libs/alibxx/crypto/base64.hpp> |
| 29 |
|
| 30 |
#include <utils/davix_gcloud_utils.hpp> |
| 31 |
#include <utils/davix_logger_internal.hpp> |
| 32 |
#include <utils/stringutils.hpp> |
| 33 |
|
| 34 |
#define SSTR(message) static_cast<std::ostringstream&>(std::ostringstream().flush() << message).str() |
| 35 |
|
| 36 |
namespace Davix{ |
| 37 |
|
| 38 |
namespace gcloud { |
| 39 |
|
| 40 |
std::string extract_bucket(const Uri & url) { |
| 41 |
std::string path = url.getPath(); |
| 42 |
std::string name = path.substr(1, path.find("/", 1)); |
| 43 |
if(name.compare(name.size()-1,1,"/") == 0) { |
| 44 |
name.erase(name.size()-1, name.size()); |
| 45 |
} |
| 46 |
return name; |
| 47 |
} |
| 48 |
|
| 49 |
std::string extract_path(const Uri & url) { |
| 50 |
std::string path = url.getPath(); |
| 51 |
std::size_t sep = path.find("/", 1); |
| 52 |
if(sep == std::string::npos) return ""; |
| 53 |
return path.substr(path.find("/", 1)+1, path.size()); |
| 54 |
} |
| 55 |
|
| 56 |
|
| 57 |
class CredentialsInternal { |
| 58 |
public: |
| 59 |
CredentialsInternal() {} |
| 60 |
std::string private_key; |
| 61 |
std::string client_email; |
| 62 |
}; |
| 63 |
|
| 64 |
bool Credentials::isEmpty() const { |
| 65 |
return internal->private_key.empty() && internal->client_email.empty(); |
| 66 |
} |
| 67 |
|
| 68 |
Credentials::Credentials() { |
| 69 |
internal = new CredentialsInternal(); |
| 70 |
} |
| 71 |
|
| 72 |
// Destructor |
| 73 |
Credentials::~Credentials() { |
| 74 |
delete internal; |
| 75 |
internal = NULL; |
| 76 |
} |
| 77 |
|
| 78 |
// Copy constructor |
| 79 |
Credentials::Credentials(const Credentials& other) { |
| 80 |
internal = new CredentialsInternal(*other.internal); |
| 81 |
} |
| 82 |
|
| 83 |
// Move constructor |
| 84 |
Credentials::Credentials(Credentials&& other) { |
| 85 |
internal = other.internal; |
| 86 |
other.internal = new CredentialsInternal(); |
| 87 |
|
| 88 |
} |
| 89 |
|
| 90 |
// Copy assignment operator |
| 91 |
Credentials& Credentials::operator=(const Credentials& other) { |
| 92 |
internal = new CredentialsInternal(*other.internal); |
| 93 |
return *this; |
| 94 |
} |
| 95 |
|
| 96 |
// Move assignment operator |
| 97 |
Credentials& Credentials::operator=(Credentials&& other) { |
| 98 |
internal = other.internal; |
| 99 |
other.internal = new CredentialsInternal(); |
| 100 |
return *this; |
| 101 |
} |
| 102 |
|
| 103 |
void Credentials::setPrivateKey(const std::string &str) { |
| 104 |
internal->private_key = str; |
| 105 |
} |
| 106 |
|
| 107 |
std::string Credentials::getPrivateKey() const { |
| 108 |
return internal->private_key; |
| 109 |
} |
| 110 |
|
| 111 |
void Credentials::setClientEmail(const std::string &str) { |
| 112 |
internal->client_email = str; |
| 113 |
} |
| 114 |
|
| 115 |
std::string Credentials::getClientEmail() const { |
| 116 |
return internal->client_email; |
| 117 |
} |
| 118 |
|
| 119 |
CredentialProvider::CredentialProvider() {} |
| 120 |
|
| 121 |
Credentials CredentialProvider::fromJSONString(const std::string &str) { |
| 122 |
Credentials creds; |
| 123 |
|
| 124 |
rapidjson::Document document; |
| 125 |
if(document.Parse(str.c_str()).HasParseError()) { |
| 126 |
throw DavixException(std::string("davix::gcloud"), StatusCode::ParsingError, "Error during JSON parsing"); |
| 127 |
} |
| 128 |
|
| 129 |
if(!document.HasMember("private_key")) { |
| 130 |
throw DavixException(std::string("davix::gcloud"), StatusCode::ParsingError, "Error during JSON parsing: Could not find private_key"); |
| 131 |
} |
| 132 |
|
| 133 |
if(!document.HasMember("client_email")) { |
| 134 |
throw DavixException(std::string("davix::gcloud"), StatusCode::ParsingError, "Error during JSON parsing: Could not find client_email"); |
| 135 |
} |
| 136 |
|
| 137 |
creds.setPrivateKey(document["private_key"].GetString()); |
| 138 |
creds.setClientEmail(document["client_email"].GetString()); |
| 139 |
|
| 140 |
return creds; |
| 141 |
} |
| 142 |
|
| 143 |
Credentials CredentialProvider::fromFile(const std::string &path) { |
| 144 |
std::stringstream buffer; |
| 145 |
|
| 146 |
try { |
| 147 |
std::ifstream tmp(path); |
| 148 |
tmp.exceptions(std::ifstream::failbit | std::ifstream::badbit); |
| 149 |
buffer << tmp.rdbuf(); |
| 150 |
} |
| 151 |
catch(...) { |
| 152 |
throw DavixException(std::string("davix::gcloud"), StatusCode::FileNotFound, SSTR("Could not read gcloud credentials at '" << path << "'")); |
| 153 |
} |
| 154 |
|
| 155 |
return fromJSONString(buffer.str()); |
| 156 |
} |
| 157 |
|
| 158 |
Uri signURIFixedTimeout(const Credentials& creds, |
| 159 |
const std::string &verb, |
| 160 |
const Uri &url, |
| 161 |
const HeaderVec &headers, |
| 162 |
const time_t signDuration) { |
| 163 |
return signURI(creds, verb, url, headers, signDuration); |
| 164 |
} |
| 165 |
|
| 166 |
Uri signURI(const Credentials& creds, |
| 167 |
const std::string &verb, |
| 168 |
const Uri &url, |
| 169 |
const HeaderVec &headers, |
| 170 |
const time_t expirationTime) { |
| 171 |
|
| 172 |
// Reference: https://cloud.google.com/storage/docs/access-control/create-signed-urls-program |
| 173 |
|
| 174 |
// Step 1: Build canonical request |
| 175 |
std::ostringstream cr; |
| 176 |
time_t currentTime = std::time(nullptr); |
| 177 |
|
| 178 |
// As of 2024-08, verbs accepted by Google are DELETE, GET, HEAD, POST, PUT |
| 179 |
// No further check here for validity of verbs in case Google changes their mind |
| 180 |
cr << verb << '\n'; |
| 181 |
|
| 182 |
// Relative path without hostname |
| 183 |
cr << url.getPath() << '\n'; |
| 184 |
|
| 185 |
// Mandatory parameters |
| 186 |
// 1: Two possible algorithms, GOOG4-HMAC-SHA256 and GOOG4-RSA-SHA256. We can force the RSA |
| 187 |
cr << "X-Goog-Algorithm=GOOG4-RSA-SHA256&"; |
| 188 |
// 2: Extract and correctly encode the credential with automatic timed scoping of the request |
| 189 |
cr << "X-Goog-Credential=" << Uri::queryParamEscape(creds.getClientEmail()) |
| 190 |
<< "%2F" |
| 191 |
<< std::put_time(std::gmtime(¤tTime), "%Y%m%d") |
| 192 |
<< Uri::queryParamEscape("/auto/storage/goog4_request") << "&"; |
| 193 |
// 3: Timestamp for good measure |
| 194 |
cr << "X-Goog-Date=" << std::put_time(std::gmtime(¤tTime), "%Y%m%dT%H%M%SZ") << "&"; |
| 195 |
// 4: Duration in seconds for request to get accepted |
| 196 |
cr << "X-Goog-Expires=" << expirationTime << "&"; |
| 197 |
// 5: We only have one signed header, the hostname of the loadbalancer |
| 198 |
cr << "X-Goog-SignedHeaders=host" << '\n'; |
| 199 |
// 6: Put the host header here |
| 200 |
cr << "host:" << url.getHost() << '\n'; |
| 201 |
|
| 202 |
// For some reason an extra blank line needs to be here |
| 203 |
cr << '\n'; |
| 204 |
|
| 205 |
// Signed header, again for whatever reason |
| 206 |
cr << "host" << '\n'; |
| 207 |
|
| 208 |
// Allow all content in the payload |
| 209 |
cr << "UNSIGNED-PAYLOAD"; |
| 210 |
|
| 211 |
// Step 2: Hash the canonical request and convert into hex |
| 212 |
std::ostringstream cr_hash, cr_hash_hexify; |
| 213 |
cr_hash << sha256(cr.str()); |
| 214 |
for (unsigned char c : cr_hash.str()) { |
| 215 |
cr_hash_hexify << std::hex << std::setw(2) << std::setfill('0') << static_cast<int>(c); |
| 216 |
} |
| 217 |
|
| 218 |
DAVIX_SLOG(DAVIX_LOG_DEBUG, DAVIX_LOG_S3, "Canonical request: {}", StrUtil::stringReplace(cr.str(), "\n", "\\n")); |
| 219 |
|
| 220 |
// Step 3: Construct the string to sign |
| 221 |
std::ostringstream ss, ss_time; |
| 222 |
ss << "GOOG4-RSA-SHA256" << '\n'; |
| 223 |
ss_time << std::put_time(std::gmtime(¤tTime), "%Y%m%dT%H%M%SZ"); |
| 224 |
ss << ss_time.str() << '\n'; |
| 225 |
ss << ss_time.str().substr(0, 8) + "/auto/storage/goog4_request" << '\n'; |
| 226 |
ss << cr_hash_hexify.str(); |
| 227 |
|
| 228 |
DAVIX_SLOG(DAVIX_LOG_DEBUG, DAVIX_LOG_S3, "String to sign: {}", StrUtil::stringReplace(ss.str(), "\n", "\\n")); |
| 229 |
|
| 230 |
// Step 4: Sign the string and convert into hex |
| 231 |
std::string binarySignature = rsasha256(creds.getPrivateKey(), ss.str()); |
| 232 |
std::ostringstream finalHex; |
| 233 |
for (unsigned char c : binarySignature) { |
| 234 |
finalHex << std::hex << std::setw(2) << std::setfill('0') << static_cast<int>(c); |
| 235 |
} |
| 236 |
|
| 237 |
// Step 5: Signed URL must have the same set of query parameters as the original canonical request |
| 238 |
// But be careful with parameter escaping |
| 239 |
Uri signedUrl(url); |
| 240 |
|
| 241 |
// Same fixed algorithm type and credential |
| 242 |
signedUrl.addQueryParam("X-Goog-Algorithm", "GOOG4-RSA-SHA256"); |
| 243 |
std::ostringstream googCred; |
| 244 |
googCred << creds.getClientEmail() << "/" << std::put_time(std::gmtime(¤tTime), "%Y%m%d") << "/auto/storage/goog4_request"; |
| 245 |
signedUrl.addQueryParam("X-Goog-Credential", googCred.str()); |
| 246 |
|
| 247 |
// Same timestamp and duration |
| 248 |
std::ostringstream googDate; |
| 249 |
googDate << std::put_time(std::gmtime(¤tTime), "%Y%m%dT%H%M%SZ"); |
| 250 |
signedUrl.addQueryParam("X-Goog-Date", googDate.str()); |
| 251 |
signedUrl.addQueryParam("X-Goog-Expires", std::to_string(expirationTime)); |
| 252 |
|
| 253 |
// Signed headers again |
| 254 |
signedUrl.addQueryParam("X-Goog-SignedHeaders", "host"); |
| 255 |
|
| 256 |
// And finally the actual signature |
| 257 |
signedUrl.addQueryParam("X-Goog-Signature", finalHex.str()); |
| 258 |
|
| 259 |
// Step 6: Done |
| 260 |
return signedUrl; |
| 261 |
} |
| 262 |
|
| 263 |
Uri getListingURI(const Uri & original_url, const RequestParams & params) { |
| 264 |
Uri newUri = original_url; |
| 265 |
newUri.setPath("/" + extract_bucket(original_url) ); |
| 266 |
|
| 267 |
std::string filename = extract_path(original_url); |
| 268 |
if(filename[filename.size()-1] != '/') { |
| 269 |
filename.append("/"); |
| 270 |
} |
| 271 |
|
| 272 |
// special case: listing top-dir |
| 273 |
if(filename == "/") |
| 274 |
filename = ""; |
| 275 |
|
| 276 |
newUri.addQueryParam("prefix", filename); |
| 277 |
newUri.addQueryParam("delimiter", "/"); |
| 278 |
newUri.addQueryParam("max-keys", "1000000000"); |
| 279 |
|
| 280 |
return newUri; |
| 281 |
} |
| 282 |
|
| 283 |
} // namespace gcloud |
| 284 |
} // namespace Davix |