Debian Codegraph @

src/utils/davix_gcloud_utils.cpp

raw
1 /*
2 * This File is part of Davix, The IO library for HTTP based protocols
3 * Copyright (C) CERN 2018
4 * Authors: Georgios Bitzes <[email protected]>
5 * Mario Lassnig <[email protected]>
6 *
7 * This library is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU Lesser General Public
9 * License as published by the Free Software Foundation; either
10 * version 2.1 of the License, or (at your option) any later version.
11 *
12 * This library is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 * Lesser General Public License for more details.
16 *
17 * You should have received a copy of the GNU Lesser General Public
18 * License along with this library; if not, write to the Free Software
19 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
20 *
21 */
22
23 #include <fstream>
24 #include <iomanip>
25 #include <sstream>
26 #include <rapidjson/document.h>
27 #include <libs/alibxx/crypto/hmacsha.hpp>
28 #include <libs/alibxx/crypto/base64.hpp>
29
30 #include <utils/davix_gcloud_utils.hpp>
31 #include <utils/davix_logger_internal.hpp>
32 #include <utils/stringutils.hpp>
33
34 #define SSTR(message) static_cast<std::ostringstream&>(std::ostringstream().flush() << message).str()
35
36 namespace Davix{
37
38 namespace gcloud {
39
40 std::string extract_bucket(const Uri & url) {
41 std::string path = url.getPath();
42 std::string name = path.substr(1, path.find("/", 1));
43 if(name.compare(name.size()-1,1,"/") == 0) {
44 name.erase(name.size()-1, name.size());
45 }
46 return name;
47 }
48
49 std::string extract_path(const Uri & url) {
50 std::string path = url.getPath();
51 std::size_t sep = path.find("/", 1);
52 if(sep == std::string::npos) return "";
53 return path.substr(path.find("/", 1)+1, path.size());
54 }
55
56
57 class CredentialsInternal {
58 public:
59 CredentialsInternal() {}
60 std::string private_key;
61 std::string client_email;
62 };
63
64 bool Credentials::isEmpty() const {
65 return internal->private_key.empty() && internal->client_email.empty();
66 }
67
68 Credentials::Credentials() {
69 internal = new CredentialsInternal();
70 }
71
72 // Destructor
73 Credentials::~Credentials() {
74 delete internal;
75 internal = NULL;
76 }
77
78 // Copy constructor
79 Credentials::Credentials(const Credentials& other) {
80 internal = new CredentialsInternal(*other.internal);
81 }
82
83 // Move constructor
84 Credentials::Credentials(Credentials&& other) {
85 internal = other.internal;
86 other.internal = new CredentialsInternal();
87
88 }
89
90 // Copy assignment operator
91 Credentials& Credentials::operator=(const Credentials& other) {
92 internal = new CredentialsInternal(*other.internal);
93 return *this;
94 }
95
96 // Move assignment operator
97 Credentials& Credentials::operator=(Credentials&& other) {
98 internal = other.internal;
99 other.internal = new CredentialsInternal();
100 return *this;
101 }
102
103 void Credentials::setPrivateKey(const std::string &str) {
104 internal->private_key = str;
105 }
106
107 std::string Credentials::getPrivateKey() const {
108 return internal->private_key;
109 }
110
111 void Credentials::setClientEmail(const std::string &str) {
112 internal->client_email = str;
113 }
114
115 std::string Credentials::getClientEmail() const {
116 return internal->client_email;
117 }
118
119 CredentialProvider::CredentialProvider() {}
120
121 Credentials CredentialProvider::fromJSONString(const std::string &str) {
122 Credentials creds;
123
124 rapidjson::Document document;
125 if(document.Parse(str.c_str()).HasParseError()) {
126 throw DavixException(std::string("davix::gcloud"), StatusCode::ParsingError, "Error during JSON parsing");
127 }
128
129 if(!document.HasMember("private_key")) {
130 throw DavixException(std::string("davix::gcloud"), StatusCode::ParsingError, "Error during JSON parsing: Could not find private_key");
131 }
132
133 if(!document.HasMember("client_email")) {
134 throw DavixException(std::string("davix::gcloud"), StatusCode::ParsingError, "Error during JSON parsing: Could not find client_email");
135 }
136
137 creds.setPrivateKey(document["private_key"].GetString());
138 creds.setClientEmail(document["client_email"].GetString());
139
140 return creds;
141 }
142
143 Credentials CredentialProvider::fromFile(const std::string &path) {
144 std::stringstream buffer;
145
146 try {
147 std::ifstream tmp(path);
148 tmp.exceptions(std::ifstream::failbit | std::ifstream::badbit);
149 buffer << tmp.rdbuf();
150 }
151 catch(...) {
152 throw DavixException(std::string("davix::gcloud"), StatusCode::FileNotFound, SSTR("Could not read gcloud credentials at '" << path << "'"));
153 }
154
155 return fromJSONString(buffer.str());
156 }
157
158 Uri signURIFixedTimeout(const Credentials& creds,
159 const std::string &verb,
160 const Uri &url,
161 const HeaderVec &headers,
162 const time_t signDuration) {
163 return signURI(creds, verb, url, headers, signDuration);
164 }
165
166 Uri signURI(const Credentials& creds,
167 const std::string &verb,
168 const Uri &url,
169 const HeaderVec &headers,
170 const time_t expirationTime) {
171
172 // Reference: https://cloud.google.com/storage/docs/access-control/create-signed-urls-program
173
174 // Step 1: Build canonical request
175 std::ostringstream cr;
176 time_t currentTime = std::time(nullptr);
177
178 // As of 2024-08, verbs accepted by Google are DELETE, GET, HEAD, POST, PUT
179 // No further check here for validity of verbs in case Google changes their mind
180 cr << verb << '\n';
181
182 // Relative path without hostname
183 cr << url.getPath() << '\n';
184
185 // Mandatory parameters
186 // 1: Two possible algorithms, GOOG4-HMAC-SHA256 and GOOG4-RSA-SHA256. We can force the RSA
187 cr << "X-Goog-Algorithm=GOOG4-RSA-SHA256&";
188 // 2: Extract and correctly encode the credential with automatic timed scoping of the request
189 cr << "X-Goog-Credential=" << Uri::queryParamEscape(creds.getClientEmail())
190 << "%2F"
191 << std::put_time(std::gmtime(&currentTime), "%Y%m%d")
192 << Uri::queryParamEscape("/auto/storage/goog4_request") << "&";
193 // 3: Timestamp for good measure
194 cr << "X-Goog-Date=" << std::put_time(std::gmtime(&currentTime), "%Y%m%dT%H%M%SZ") << "&";
195 // 4: Duration in seconds for request to get accepted
196 cr << "X-Goog-Expires=" << expirationTime << "&";
197 // 5: We only have one signed header, the hostname of the loadbalancer
198 cr << "X-Goog-SignedHeaders=host" << '\n';
199 // 6: Put the host header here
200 cr << "host:" << url.getHost() << '\n';
201
202 // For some reason an extra blank line needs to be here
203 cr << '\n';
204
205 // Signed header, again for whatever reason
206 cr << "host" << '\n';
207
208 // Allow all content in the payload
209 cr << "UNSIGNED-PAYLOAD";
210
211 // Step 2: Hash the canonical request and convert into hex
212 std::ostringstream cr_hash, cr_hash_hexify;
213 cr_hash << sha256(cr.str());
214 for (unsigned char c : cr_hash.str()) {
215 cr_hash_hexify << std::hex << std::setw(2) << std::setfill('0') << static_cast<int>(c);
216 }
217
218 DAVIX_SLOG(DAVIX_LOG_DEBUG, DAVIX_LOG_S3, "Canonical request: {}", StrUtil::stringReplace(cr.str(), "\n", "\\n"));
219
220 // Step 3: Construct the string to sign
221 std::ostringstream ss, ss_time;
222 ss << "GOOG4-RSA-SHA256" << '\n';
223 ss_time << std::put_time(std::gmtime(&currentTime), "%Y%m%dT%H%M%SZ");
224 ss << ss_time.str() << '\n';
225 ss << ss_time.str().substr(0, 8) + "/auto/storage/goog4_request" << '\n';
226 ss << cr_hash_hexify.str();
227
228 DAVIX_SLOG(DAVIX_LOG_DEBUG, DAVIX_LOG_S3, "String to sign: {}", StrUtil::stringReplace(ss.str(), "\n", "\\n"));
229
230 // Step 4: Sign the string and convert into hex
231 std::string binarySignature = rsasha256(creds.getPrivateKey(), ss.str());
232 std::ostringstream finalHex;
233 for (unsigned char c : binarySignature) {
234 finalHex << std::hex << std::setw(2) << std::setfill('0') << static_cast<int>(c);
235 }
236
237 // Step 5: Signed URL must have the same set of query parameters as the original canonical request
238 // But be careful with parameter escaping
239 Uri signedUrl(url);
240
241 // Same fixed algorithm type and credential
242 signedUrl.addQueryParam("X-Goog-Algorithm", "GOOG4-RSA-SHA256");
243 std::ostringstream googCred;
244 googCred << creds.getClientEmail() << "/" << std::put_time(std::gmtime(&currentTime), "%Y%m%d") << "/auto/storage/goog4_request";
245 signedUrl.addQueryParam("X-Goog-Credential", googCred.str());
246
247 // Same timestamp and duration
248 std::ostringstream googDate;
249 googDate << std::put_time(std::gmtime(&currentTime), "%Y%m%dT%H%M%SZ");
250 signedUrl.addQueryParam("X-Goog-Date", googDate.str());
251 signedUrl.addQueryParam("X-Goog-Expires", std::to_string(expirationTime));
252
253 // Signed headers again
254 signedUrl.addQueryParam("X-Goog-SignedHeaders", "host");
255
256 // And finally the actual signature
257 signedUrl.addQueryParam("X-Goog-Signature", finalHex.str());
258
259 // Step 6: Done
260 return signedUrl;
261 }
262
263 Uri getListingURI(const Uri & original_url, const RequestParams & params) {
264 Uri newUri = original_url;
265 newUri.setPath("/" + extract_bucket(original_url) );
266
267 std::string filename = extract_path(original_url);
268 if(filename[filename.size()-1] != '/') {
269 filename.append("/");
270 }
271
272 // special case: listing top-dir
273 if(filename == "/")
274 filename = "";
275
276 newUri.addQueryParam("prefix", filename);
277 newUri.addQueryParam("delimiter", "/");
278 newUri.addQueryParam("max-keys", "1000000000");
279
280 return newUri;
281 }
282
283 } // namespace gcloud
284 } // namespace Davix