Debian Codegraph @

libs/matdbg/src/DebugServer.cpp

raw
1 /*
2 * Copyright (C) 2019 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #include <matdbg/DebugServer.h>
18
19 #include <CivetServer.h>
20
21 #include <utils/Hash.h>
22 #include <utils/Log.h>
23
24 #include <spirv_glsl.hpp>
25 #include <spirv-tools/libspirv.h>
26
27 #include <matdbg/ShaderReplacer.h>
28 #include <matdbg/ShaderExtractor.h>
29 #include <matdbg/ShaderInfo.h>
30 #include <matdbg/JsonWriter.h>
31
32 #include <filaflat/ChunkContainer.h>
33
34 #include <backend/DriverEnums.h>
35
36 #include <sstream>
37 #include <string>
38
39 // If set to 0, this serves HTML from a resgen resource. Use 1 only during local development, which
40 // serves files directly from the source code tree.
41 #define SERVE_FROM_SOURCE_TREE 0
42
43 #if !SERVE_FROM_SOURCE_TREE
44 #include "matdbg_resources.h"
45 #endif
46
47 namespace filament {
48 namespace matdbg {
49
50 using namespace utils;
51 using namespace filament::backend;
52
53 using filaflat::ChunkContainer;
54 using filamat::ChunkType;
55
56 static const StaticString kSuccessHeader =
57 "HTTP/1.1 200 OK\r\nContent-Type: %s\r\n"
58 "Connection: close\r\n\r\n";
59
60 class FileRequestHandler : public CivetHandler {
61 public:
62 FileRequestHandler(DebugServer* server) : mServer(server) {}
63 bool handleGet(CivetServer *server, struct mg_connection *conn) {
64 const struct mg_request_info* request = mg_get_request_info(conn);
65 std::string uri(request->request_uri);
66 if (uri == "/" || uri == "/index.html") {
67 #if SERVE_FROM_SOURCE_TREE
68 mg_send_file(conn, "libs/matdbg/web/index.html");
69 #else
70 mg_printf(conn, kSuccessHeader.c_str(), "text/html");
71 mg_write(conn, mServer->mHtml.c_str(), mServer->mHtml.size());
72 #endif
73 return true;
74 }
75 if (uri == "/style.css") {
76 #if SERVE_FROM_SOURCE_TREE
77 mg_send_file(conn, "libs/matdbg/web/style.css");
78 #else
79 mg_printf(conn, kSuccessHeader.c_str(), "text/css");
80 mg_write(conn, mServer->mCss.c_str(), mServer->mCss.size());
81 #endif
82 return true;
83 }
84 if (uri == "/script.js") {
85 #if SERVE_FROM_SOURCE_TREE
86 mg_send_file(conn, "libs/matdbg/web/script.js");
87 #else
88 mg_printf(conn, kSuccessHeader.c_str(), "text/javascript");
89 mg_write(conn, mServer->mJavascript.c_str(), mServer->mJavascript.size());
90 #endif
91 return true;
92 }
93 slog.e << "DebugServer: bad request at line " << __LINE__ << ": " << uri << io::endl;
94 return false;
95 }
96 private:
97 DebugServer* mServer;
98 };
99
100 // Handles the following REST requests, where {id} is an 8-digit hex string.
101 //
102 // GET /api/matids
103 // GET /api/materials
104 // GET /api/material?matid={id}
105 // GET /api/shader?matid={id}&type=[glsl|spirv]&[glindex|vkindex|metalindex]={index}
106 // GET /api/active
107 //
108 class RestRequestHandler : public CivetHandler {
109 public:
110 RestRequestHandler(DebugServer* server) : mServer(server) {}
111
112 bool handleGet(CivetServer *server, struct mg_connection *conn) {
113 const struct mg_request_info* request = mg_get_request_info(conn);
114 std::string uri(request->local_uri);
115
116 const auto error = [request](int line) {
117 slog.e << "DebugServer: 404 at " << line << ": " << request->query_string << io::endl;
118 return false;
119 };
120
121 if (uri == "/api/active") {
122 mServer->updateActiveVariants();
123 mg_printf(conn, kSuccessHeader.c_str(), "application/json");
124 mg_printf(conn, "{");
125 int index = 0;
126 for (const auto& pair : mServer->mMaterialRecords) {
127 const auto& record = pair.second;
128 ChunkContainer package(record.package, record.packageSize);
129 if (!package.parse()) {
130 return error(__LINE__);
131 }
132 JsonWriter writer;
133 if (!writer.writeActiveInfo(package, mServer->mBackend, record.activeVariants)) {
134 return error(__LINE__);
135 }
136 const bool last = (++index) == mServer->mMaterialRecords.size();
137 mg_printf(conn, "\"%8.8x\": %s %s", pair.first, writer.getJsonString(),
138 last ? "" : ",");
139 }
140 mg_printf(conn, "}");
141 return true;
142 }
143
144 if (uri == "/api/matids") {
145 mg_printf(conn, kSuccessHeader.c_str(), "application/json");
146 mg_printf(conn, "[");
147 int index = 0;
148 for (const auto& record : mServer->mMaterialRecords) {
149 const bool last = (++index) == mServer->mMaterialRecords.size();
150 mg_printf(conn, "\"%8.8x\" %s", record.first, last ? "" : ",");
151 }
152 mg_printf(conn, "]");
153 return true;
154 }
155
156 if (uri == "/api/materials") {
157 mg_printf(conn, kSuccessHeader.c_str(), "application/json");
158 mg_printf(conn, "[");
159 int index = 0;
160 for (const auto& record : mServer->mMaterialRecords) {
161 const bool last = (++index) == mServer->mMaterialRecords.size();
162
163 ChunkContainer package(record.second.package, record.second.packageSize);
164 if (!package.parse()) {
165 return error(__LINE__);
166 }
167
168 JsonWriter writer;
169 if (!writer.writeMaterialInfo(package)) {
170 return error(__LINE__);
171 }
172
173 mg_printf(conn, "{ \"matid\": \"%8.8x\", %s } %s", record.first,
174 writer.getJsonString(), last ? "" : ",");
175 }
176 mg_printf(conn, "]");
177 return true;
178 }
179
180 if (!request->query_string) {
181 return error(__LINE__);
182 }
183
184 const size_t qlength = strlen(request->query_string);
185 char matid[9] = {};
186 if (mg_get_var(request->query_string, qlength, "matid", matid, sizeof(matid)) < 0) {
187 return error(__LINE__);
188 }
189 const uint32_t id = strtol(matid, nullptr, 16);
190 const DebugServer::MaterialRecord* result = mServer->getRecord(id);
191 if (result == nullptr) {
192 return error(__LINE__);
193 }
194
195 ChunkContainer package(result->package, result->packageSize);
196 if (!package.parse()) {
197 return error(__LINE__);
198 }
199
200 if (uri == "/api/material") {
201 JsonWriter writer;
202 if (!writer.writeMaterialInfo(package)) {
203 return error(__LINE__);
204 }
205 mg_printf(conn, kSuccessHeader.c_str(), "application/json");
206 mg_printf(conn, "{ %s }", writer.getJsonString());
207 return true;
208 }
209
210 char type[6] = {};
211 if (mg_get_var(request->query_string, qlength, "type", type, sizeof(type)) < 0) {
212 return error(__LINE__);
213 }
214
215 char glindex[4] = {};
216 char vkindex[4] = {};
217 char metalindex[4] = {};
218 mg_get_var(request->query_string, qlength, "glindex", glindex, sizeof(glindex));
219 mg_get_var(request->query_string, qlength, "vkindex", vkindex, sizeof(vkindex));
220 mg_get_var(request->query_string, qlength, "metalindex", metalindex, sizeof(metalindex));
221
222 if (!glindex[0] && !vkindex[0] && !metalindex[0]) {
223 return error(__LINE__);
224 }
225
226 if (uri != "/api/shader") {
227 return error(__LINE__);
228 }
229
230 if (glindex[0]) {
231 std::vector<ShaderInfo> info(getShaderCount(package, ChunkType::MaterialGlsl));
232 if (!getGlShaderInfo(package, info.data())) {
233 return error(__LINE__);
234 }
235
236 const int shaderIndex = std::stoi(glindex);
237 if (shaderIndex >= info.size()) {
238 return error(__LINE__);
239 }
240
241 ShaderExtractor extractor(Backend::OPENGL, result->package, result->packageSize);
242 if (!extractor.parse()) {
243 return error(__LINE__);
244 }
245
246 const auto& item = info[shaderIndex];
247 filaflat::ShaderBuilder builder;
248 extractor.getShader(item.shaderModel, item.variant, item.pipelineStage, builder);
249
250 mg_printf(conn, kSuccessHeader.c_str(), "application/txt");
251 mg_write(conn, builder.data(), builder.size() - 1);
252 return true;
253 }
254
255 if (vkindex[0]) {
256 ShaderExtractor extractor(Backend::VULKAN, result->package, result->packageSize);
257 if (!extractor.parse()) {
258 return error(__LINE__);
259 }
260
261 filaflat::ShaderBuilder builder;
262 std::vector<ShaderInfo> info(getShaderCount(package, ChunkType::MaterialSpirv));
263 if (!getVkShaderInfo(package, info.data())) {
264 return error(__LINE__);
265 }
266
267 const int shaderIndex = std::stoi(vkindex);
268 if (shaderIndex >= info.size()) {
269 return error(__LINE__);
270 }
271
272 const auto& item = info[shaderIndex];
273 extractor.getShader(item.shaderModel, item.variant, item.pipelineStage, builder);
274
275 // TODO: Add a transpiler that depends on "type" and add an MSL type instead of
276 // piggybacking on type=GLSL.
277
278 mg_printf(conn, kSuccessHeader.c_str(), "application/text");
279
280 if (true) {
281 auto context = spvContextCreate(SPV_ENV_UNIVERSAL_1_0);
282 spv_text text = nullptr;
283 const uint32_t options = SPV_BINARY_TO_TEXT_OPTION_INDENT |
284 SPV_BINARY_TO_TEXT_OPTION_FRIENDLY_NAMES;
285 const uint32_t* data = (const uint32_t*) builder.data();
286 spvBinaryToText(context, data, builder.size() / 4, options, &text, nullptr);
287
288 mg_write(conn, text->str, text->length);
289 spvTextDestroy(text);
290 spvContextDestroy(context);
291 }
292
293 return true;
294 }
295
296 if (metalindex[0]) {
297 ShaderExtractor extractor(Backend::METAL, result->package, result->packageSize);
298 if (!extractor.parse()) {
299 return error(__LINE__);
300 }
301
302 filaflat::ShaderBuilder builder;
303 std::vector<ShaderInfo> info(getShaderCount(package, ChunkType::MaterialMetal));
304 if (!getMetalShaderInfo(package, info.data())) {
305 return error(__LINE__);
306 }
307
308 const int shaderIndex = std::stoi(metalindex);
309 if (shaderIndex >= info.size()) {
310 return error(__LINE__);
311 }
312
313 const auto& item = info[shaderIndex];
314 extractor.getShader(item.shaderModel, item.variant, item.pipelineStage, builder);
315
316 mg_printf(conn, kSuccessHeader.c_str(), "application/txt");
317 mg_write(conn, builder.data(), builder.size() - 1);
318 return true;
319 }
320
321 return error(__LINE__);
322 }
323
324 private:
325 DebugServer* mServer;
326 };
327
328 class WebSocketHandler : public CivetWebSocketHandler {
329 public:
330 WebSocketHandler(DebugServer* server) : mServer(server) {}
331
332 bool handleConnection(CivetServer *server, const struct mg_connection *conn) override {
333 return true;
334 }
335
336 void handleReadyState(CivetServer *server, struct mg_connection *conn) override {
337 mConnection = conn;
338 }
339
340 bool handleData(CivetServer *server, struct mg_connection *conn, int bits, char *data,
341 size_t size) override {
342
343 // First check if this chunk is a continuation of a partial existing message.
344 if (mServer->mChunkedMessageRemaining > 0) {
345 const CString chunk(data, size);
346 const size_t pos = mServer->mChunkedMessage.size();
347
348 // Append the partial existing message.
349 mServer->mChunkedMessage = mServer->mChunkedMessage.insert(pos, chunk);
350
351 // Determine number of outstanding bytes.
352 if (size > mServer->mChunkedMessageRemaining) {
353 mServer->mChunkedMessageRemaining = 0;
354 } else {
355 mServer->mChunkedMessageRemaining -= size;
356 }
357
358 // Return early and wait for more chunks if some bytes are still outstanding.
359 if (mServer->mChunkedMessageRemaining > 0) {
360 return true;
361 }
362
363 data = mServer->mChunkedMessage.data();
364 size = mServer->mChunkedMessage.size();
365
366 // Ignore the handshake message that occurs after startup.
367 } else if (size < 8) {
368 return true;
369 }
370
371 mServer->mChunkedMessageRemaining = 0;
372
373 // Every WebSocket message is prefixed with a command name followed by a space.
374 //
375 // For now we simply use istringstream for parsing, so command arguments are delimited
376 // with space characters.
377 //
378 // The "API index" matches the values of filament::backend::Backend (zero is invalid).
379 //
380 // The "shader index" is a zero-based index into the list of variants using the order that
381 // they appear in the package, where each API (GL / VK / Metal) has its own list.
382 //
383 // Commands:
384 //
385 // EDIT [material id] [api index] [shader index] [shader length] [shader source....]
386 //
387
388 const static StaticString kEditCmd = "EDIT ";
389 const static size_t kEditCmdLength = kEditCmd.size();
390
391 if (0 == strncmp(data, kEditCmd.c_str(), kEditCmdLength)) {
392 std::istringstream str(data + kEditCmdLength);
393 uint32_t matid;
394 int api;
395 int shaderIndex;
396 int shaderLength;
397 str >> std::hex >> matid >> std::dec >> api >> shaderIndex >> shaderLength;
398 const char* source = data + kEditCmdLength + str.tellg() + 1;
399 const size_t remaining = size - kEditCmdLength - str.tellg();
400
401 // Return early and wait for more chunks if some bytes are still outstanding.
402 if (remaining < shaderLength + 1) {
403 mServer->mChunkedMessage = CString(data, size);
404 mServer->mChunkedMessageRemaining = shaderLength + 1 - remaining;
405 return true;
406 }
407
408 mServer->handleEditCommand(matid, backend::Backend(api), shaderIndex, source,
409 shaderLength);
410 return true;
411 }
412
413 const std::string firstFewChars(data, std::min(size, size_t(8)));
414 slog.e << "Bad WebSocket message. First few characters: "
415 << "[" << firstFewChars << "]" << io::endl;
416 return false;
417 }
418
419 void handleClose(CivetServer *server, const struct mg_connection *conn) override {
420 mConnection = nullptr;
421 }
422
423 // Notify the JavaScript client that a new material package has been loaded.
424 void addMaterial(const DebugServer::MaterialRecord& material) {
425 if (mConnection) {
426 char matid[9] = {};
427 snprintf(matid, sizeof(matid), "%8.8x", material.key);
428 mg_websocket_write(mConnection, MG_WEBSOCKET_OPCODE_TEXT, matid, 8);
429 }
430 }
431
432 private:
433 DebugServer* mServer;
434 struct mg_connection* mConnection = nullptr;
435 };
436
437 DebugServer::DebugServer(Backend backend, int port) : mBackend(backend) {
438 #if !SERVE_FROM_SOURCE_TREE
439 mHtml = CString((const char*) MATDBG_RESOURCES_INDEX_DATA, MATDBG_RESOURCES_INDEX_SIZE - 1);
440 mJavascript = CString((const char*) MATDBG_RESOURCES_SCRIPT_DATA, MATDBG_RESOURCES_SCRIPT_SIZE - 1);
441 mCss = CString((const char*) MATDBG_RESOURCES_STYLE_DATA, MATDBG_RESOURCES_STYLE_SIZE - 1);
442 #endif
443
444 // By default the server spawns 50 threads so we override this to 2. This limits the server
445 // to having no more than 2 HTTP clients, which is perfectly fine for debugging purposes.
446 const char* kServerOptions[] = {
447 "listening_ports", "8080",
448 "num_threads", "2",
449 "error_log_file", "civetweb.txt",
450 nullptr
451 };
452 std::string portString = std::to_string(port);
453 kServerOptions[1] = portString.c_str();
454
455 mServer = new CivetServer(kServerOptions);
456 if (!mServer->getContext()) {
457 delete mServer;
458 mServer = nullptr;
459 slog.e << "Unable to start DebugServer, see civetweb.txt for details." << io::endl;
460 return;
461 }
462
463 mFileHandler = new FileRequestHandler(this);
464 mRestHandler = new RestRequestHandler(this);
465 mWebSocketHandler = new WebSocketHandler(this);
466
467 mServer->addHandler("/api", mRestHandler);
468 mServer->addHandler("", mFileHandler);
469 mServer->addWebSocketHandler("", mWebSocketHandler);
470
471 slog.i << "DebugServer listening at http://localhost:" << port << io::endl;
472 }
473
474 DebugServer::~DebugServer() {
475 for (auto& pair : mMaterialRecords) {
476 delete [] pair.second.package;
477 }
478 delete mFileHandler;
479 delete mRestHandler;
480 delete mServer;
481 }
482
483 MaterialKey
484 DebugServer::addMaterial(const CString& name, const void* data, size_t size, void* userdata) {
485 filaflat::ChunkContainer* container = new filaflat::ChunkContainer(data, size);
486 if (!container->parse()) {
487 slog.e << "DebugServer: unable to parse material package: " << name.c_str() << io::endl;
488 return {};
489 }
490
491 const uint32_t seed = 42;
492 auto words = (const uint32_t*) data;
493 MaterialKey key = utils::hash::murmur3(words, size / 4, seed);
494
495 // Retain a copy of the package to permit queries after the client application has
496 // freed up the original material package.
497 uint8_t* package = new uint8_t[size];
498 memcpy(package, data, size);
499
500 MaterialRecord info = {userdata, package, size, name, key};
501 mMaterialRecords.insert({key, info});
502 mWebSocketHandler->addMaterial(info);
503 return key;
504 }
505
506 void DebugServer::removeMaterial(MaterialKey key) {
507 mMaterialRecords.erase(key);
508 }
509
510 const DebugServer::MaterialRecord* DebugServer::getRecord(const MaterialKey& key) const {
511 const auto& iter = mMaterialRecords.find(key);
512 return iter == mMaterialRecords.end() ? nullptr : &iter->second;
513 }
514
515 void DebugServer::updateActiveVariants() {
516 if (mQueryCallback) {
517 for (auto& pair : mMaterialRecords) {
518 uint64_t& result = mMaterialRecords[pair.first].activeVariants;
519 mQueryCallback(pair.second.userdata, &result);
520 }
521 }
522 }
523
524 bool DebugServer::handleEditCommand(const MaterialKey& key, backend::Backend api, int shaderIndex,
525 const char* source, size_t size) {
526 const auto error = [](int line) {
527 slog.e << "DebugServer: Unable to apply shader edit at line " << line << io::endl;
528 return false;
529 };
530
531 if (mMaterialRecords.find(key) == mMaterialRecords.end()) {
532 return error(__LINE__);
533 }
534 MaterialRecord& material = mMaterialRecords[key];
535 filaflat::ChunkContainer package(material.package, material.packageSize);
536 if (!package.parse()) {
537 return error(__LINE__);
538 }
539
540 std::vector<ShaderInfo> infos;
541 size_t shaderCount;
542 switch (api) {
543 case backend::Backend::OPENGL: {
544 shaderCount = getShaderCount(package, ChunkType::MaterialGlsl);
545 infos.resize(shaderCount);
546 if (!getGlShaderInfo(package, infos.data())) {
547 return error(__LINE__);
548 }
549 break;
550 }
551 case backend::Backend::VULKAN: {
552 shaderCount = getShaderCount(package, ChunkType::MaterialSpirv);
553 infos.resize(shaderCount);
554 if (!getVkShaderInfo(package, infos.data())) {
555 return error(__LINE__);
556 }
557 break;
558 }
559 case backend::Backend::METAL: {
560 shaderCount = getShaderCount(package, ChunkType::MaterialMetal);
561 infos.resize(shaderCount);
562 if (!getMetalShaderInfo(package, infos.data())) {
563 return error(__LINE__);
564 }
565 break;
566 }
567 default:
568 error(__LINE__);
569 }
570
571 if (shaderIndex < 0 || shaderIndex >= shaderCount) {
572 return error(__LINE__);
573 }
574
575 const ShaderInfo info = infos[shaderIndex];
576 ShaderReplacer editor(api, package.getData(), package.getSize());
577 if (!editor.replaceShaderSource(info.shaderModel, info.variant, info.pipelineStage, source, size)) {
578 return error(__LINE__);
579 }
580
581 delete [] material.package;
582
583 material.package = editor.getEditedPackage();
584 material.packageSize = editor.getEditedSize();
585
586 if (mEditCallback) {
587 mEditCallback(material.userdata, material.name, material.package, material.packageSize);
588 }
589
590 return true;
591 }
592
593 } // namespace matdbg
594 } // namespace filament