Debian Codegraph @

src/gfxstream/guest/GoldfishAddressSpace/AddressSpaceStream.cpp

raw
1 /*
2 * Copyright 2020 Google
3 * SPDX-License-Identifier: MIT
4 */
5 #include "AddressSpaceStream.h"
6
7 #include <errno.h>
8 #include <stdio.h>
9 #include <stdlib.h>
10 #include <string.h>
11 #include <unistd.h>
12
13 #include "VirtGpu.h"
14 #include "util/log.h"
15 #include "util/perf/cpu_trace.h"
16
17 static const size_t kReadSize = 512 * 1024;
18 static const size_t kWriteOffset = kReadSize;
19
20 AddressSpaceStream::AddressSpaceStream(address_space_handle_t handle, uint32_t version,
21 struct asg_context context, uint64_t ringOffset,
22 uint64_t writeBufferOffset, struct address_space_ops ops)
23 : IOStream(context.ring_config->flush_interval),
24 m_ops(ops),
25 m_tmpBuf(0),
26 m_tmpBufSize(0),
27 m_tmpBufXferSize(0),
28 m_usingTmpBuf(0),
29 m_readBuf(0),
30 m_read(0),
31 m_readLeft(0),
32 m_handle(handle),
33 m_version(version),
34 m_context(context),
35 m_ringOffset(ringOffset),
36 m_writeBufferOffset(writeBufferOffset),
37 m_writeBufferSize(context.ring_config->buffer_size),
38 m_writeBufferMask(m_writeBufferSize - 1),
39 m_buf((unsigned char*)context.buffer),
40 m_writeStart(m_buf),
41 m_writeStep(context.ring_config->flush_interval),
42 m_notifs(0),
43 m_written(0),
44 m_backoffIters(0),
45 m_backoffFactor(1),
46 m_ringStorageSize(sizeof(struct asg_ring_storage) + m_writeBufferSize) {
47 // We'll use this in the future, but at the moment,
48 // it's a potential compile Werror.
49 (void)m_ringStorageSize;
50 (void)m_version;
51 }
52
53 AddressSpaceStream::~AddressSpaceStream() {
54 flush();
55 ensureType3Finished();
56 ensureType1Finished();
57
58 if (!m_mapping) {
59 m_ops.unmap(m_context.to_host, sizeof(struct asg_ring_storage));
60 m_ops.unmap(m_context.buffer, m_writeBufferSize);
61 m_ops.unclaim_shared(m_handle, m_ringOffset);
62 m_ops.unclaim_shared(m_handle, m_writeBufferOffset);
63 }
64
65 m_ops.close(m_handle);
66 if (m_readBuf) free(m_readBuf);
67 if (m_tmpBuf) free(m_tmpBuf);
68 }
69
70 size_t AddressSpaceStream::idealAllocSize(size_t len) {
71 if (len > m_writeStep) return len;
72 return m_writeStep;
73 }
74
75 void* AddressSpaceStream::allocBuffer(size_t minSize) {
76 MESA_TRACE_SCOPE("allocBuffer");
77 ensureType3Finished();
78
79 if (!m_readBuf) {
80 m_readBuf = (unsigned char*)malloc(kReadSize);
81 }
82
83 size_t allocSize =
84 (m_writeStep < minSize ? minSize : m_writeStep);
85
86 if (m_writeStep < allocSize) {
87 if (!m_tmpBuf) {
88 m_tmpBufSize = allocSize * 2;
89 m_tmpBuf = (unsigned char*)malloc(m_tmpBufSize);
90 }
91
92 if (m_tmpBufSize < allocSize) {
93 m_tmpBufSize = allocSize * 2;
94 m_tmpBuf = (unsigned char*)realloc(m_tmpBuf, m_tmpBufSize);
95 }
96
97 if (!m_usingTmpBuf) {
98 flush();
99 }
100
101 m_usingTmpBuf = true;
102 m_tmpBufXferSize = allocSize;
103 return m_tmpBuf;
104 } else {
105 if (m_usingTmpBuf) {
106 writeFully(m_tmpBuf, m_tmpBufXferSize);
107 m_usingTmpBuf = false;
108 m_tmpBufXferSize = 0;
109 }
110
111 return m_writeStart;
112 }
113 }
114
115 int AddressSpaceStream::commitBuffer(size_t size)
116 {
117 if (size == 0) return 0;
118
119 if (m_usingTmpBuf) {
120 writeFully(m_tmpBuf, size);
121 m_tmpBufXferSize = 0;
122 m_usingTmpBuf = false;
123 return 0;
124 } else {
125 int res = type1Write(m_writeStart - m_buf, size);
126 advanceWrite();
127 return res;
128 }
129 }
130
131 const unsigned char *AddressSpaceStream::readFully(void *ptr, size_t totalReadSize)
132 {
133
134 unsigned char* userReadBuf = static_cast<unsigned char*>(ptr);
135
136 if (!userReadBuf) {
137 if (totalReadSize > 0) {
138 mesa_loge(
139 "AddressSpaceStream::commitBufferAndReadFully failed, userReadBuf=NULL, "
140 "totalReadSize %zu, lethal"
141 " error, exiting.",
142 totalReadSize);
143 abort();
144 }
145 return nullptr;
146 }
147
148 // Advance buffered read if not yet consumed.
149 size_t remaining = totalReadSize;
150 size_t bufferedReadSize =
151 m_readLeft < remaining ? m_readLeft : remaining;
152
153 if (bufferedReadSize) {
154 memcpy(userReadBuf,
155 m_readBuf + (m_read - m_readLeft),
156 bufferedReadSize);
157 remaining -= bufferedReadSize;
158 m_readLeft -= bufferedReadSize;
159 }
160
161 if (!remaining) return userReadBuf;
162
163 // Read up to kReadSize bytes if all buffered read has been consumed.
164 size_t maxRead = m_readLeft ? 0 : kReadSize;
165 ssize_t actual = 0;
166
167 if (maxRead) {
168 actual = speculativeRead(m_readBuf, maxRead);
169
170 // Updated buffered read size.
171 if (actual > 0) {
172 m_read = m_readLeft = actual;
173 }
174
175 if (actual == 0) {
176 mesa_logd("%s: end of pipe", __FUNCTION__);
177 return NULL;
178 }
179 }
180
181 // Consume buffered read and read more if necessary.
182 while (remaining) {
183 bufferedReadSize = m_readLeft < remaining ? m_readLeft : remaining;
184 if (bufferedReadSize) {
185 memcpy(userReadBuf + (totalReadSize - remaining),
186 m_readBuf + (m_read - m_readLeft),
187 bufferedReadSize);
188 remaining -= bufferedReadSize;
189 m_readLeft -= bufferedReadSize;
190 continue;
191 }
192
193 actual = speculativeRead(m_readBuf, kReadSize);
194
195 if (actual == 0) {
196 mesa_logd("%s: Failed reading from pipe: %d", __FUNCTION__, errno);
197 return NULL;
198 }
199
200 if (actual > 0) {
201 m_read = m_readLeft = actual;
202 continue;
203 }
204 }
205
206 resetBackoff();
207 return userReadBuf;
208 }
209
210 const unsigned char *AddressSpaceStream::read(void *buf, size_t *inout_len) {
211 unsigned char* dst = (unsigned char*)buf;
212 size_t wanted = *inout_len;
213 ssize_t actual = speculativeRead(dst, wanted);
214
215 if (actual >= 0) {
216 *inout_len = actual;
217 } else {
218 return nullptr;
219 }
220
221 return (const unsigned char*)dst;
222 }
223
224 int AddressSpaceStream::writeFully(const void* buf, size_t size) {
225 MESA_TRACE_SCOPE("writeFully");
226 ensureType3Finished();
227 ensureType1Finished();
228
229 m_context.ring_config->transfer_size = size;
230 m_context.ring_config->transfer_mode = 3;
231
232 size_t sent = 0;
233 size_t preferredChunkSize = m_writeBufferSize / 4;
234 size_t chunkSize = size < preferredChunkSize ? size : preferredChunkSize;
235 const uint8_t* bufferBytes = (const uint8_t*)buf;
236
237 bool hostPinged = false;
238 while (sent < size) {
239 size_t remaining = size - sent;
240 size_t sendThisTime = remaining < chunkSize ? remaining : chunkSize;
241
242 long sentChunks =
243 ring_buffer_view_write(
244 m_context.to_host_large_xfer.ring,
245 &m_context.to_host_large_xfer.view,
246 bufferBytes + sent, sendThisTime, 1);
247
248 if (!hostPinged && *(m_context.host_state) != ASG_HOST_STATE_CAN_CONSUME &&
249 *(m_context.host_state) != ASG_HOST_STATE_RENDERING) {
250 notifyAvailable();
251 hostPinged = true;
252 }
253
254 if (sentChunks == 0) {
255 ring_buffer_yield();
256 backoff();
257 }
258
259 sent += sentChunks * sendThisTime;
260
261 if (isInError()) {
262 return -1;
263 }
264 }
265
266 bool isRenderingAfter = ASG_HOST_STATE_RENDERING == __atomic_load_n(m_context.host_state, __ATOMIC_ACQUIRE);
267
268 if (!isRenderingAfter) {
269 notifyAvailable();
270 }
271
272 ensureType3Finished();
273
274 resetBackoff();
275 m_context.ring_config->transfer_mode = 1;
276 m_written += size;
277
278 float mb = (float)m_written / 1048576.0f;
279 if (mb > 100.0f) {
280 mesa_logd("%s: %f mb in %d notifs. %f mb/notif\n", __func__, mb, m_notifs,
281 m_notifs ? mb / (float)m_notifs : 0.0f);
282 m_notifs = 0;
283 m_written = 0;
284 }
285 return 0;
286 }
287
288 int AddressSpaceStream::writeFullyAsync(const void* buf, size_t size) {
289 MESA_TRACE_SCOPE("writeFullyAsync");
290 ensureType3Finished();
291 ensureType1Finished();
292
293 __atomic_store_n(&m_context.ring_config->transfer_size, size, __ATOMIC_RELEASE);
294 m_context.ring_config->transfer_mode = 3;
295
296 size_t sent = 0;
297 size_t preferredChunkSize = m_writeBufferSize / 2;
298 size_t chunkSize = size < preferredChunkSize ? size : preferredChunkSize;
299 const uint8_t* bufferBytes = (const uint8_t*)buf;
300
301 bool pingedHost = false;
302
303 while (sent < size) {
304 size_t remaining = size - sent;
305 size_t sendThisTime = remaining < chunkSize ? remaining : chunkSize;
306
307 long sentChunks =
308 ring_buffer_view_write(
309 m_context.to_host_large_xfer.ring,
310 &m_context.to_host_large_xfer.view,
311 bufferBytes + sent, sendThisTime, 1);
312
313 uint32_t hostState = __atomic_load_n(m_context.host_state, __ATOMIC_ACQUIRE);
314
315 if (!pingedHost &&
316 hostState != ASG_HOST_STATE_CAN_CONSUME &&
317 hostState != ASG_HOST_STATE_RENDERING) {
318 pingedHost = true;
319 notifyAvailable();
320 }
321
322 if (sentChunks == 0) {
323 ring_buffer_yield();
324 backoff();
325 }
326
327 sent += sentChunks * sendThisTime;
328
329 if (isInError()) {
330 return -1;
331 }
332 }
333
334
335 bool isRenderingAfter = ASG_HOST_STATE_RENDERING == __atomic_load_n(m_context.host_state, __ATOMIC_ACQUIRE);
336
337 if (!isRenderingAfter) {
338 notifyAvailable();
339 }
340
341 resetBackoff();
342 m_context.ring_config->transfer_mode = 1;
343 m_written += size;
344
345 float mb = (float)m_written / 1048576.0f;
346 if (mb > 100.0f) {
347 mesa_logd("%s: %f mb in %d notifs. %f mb/notif\n", __func__, mb, m_notifs,
348 m_notifs ? mb / (float)m_notifs : 0.0f);
349 m_notifs = 0;
350 m_written = 0;
351 }
352 return 0;
353 }
354
355 const unsigned char *AddressSpaceStream::commitBufferAndReadFully(
356 size_t writeSize, void *userReadBufPtr, size_t totalReadSize) {
357
358 if (m_usingTmpBuf) {
359 writeFully(m_tmpBuf, writeSize);
360 m_usingTmpBuf = false;
361 m_tmpBufXferSize = 0;
362 return readFully(userReadBufPtr, totalReadSize);
363 } else {
364 commitBuffer(writeSize);
365 return readFully(userReadBufPtr, totalReadSize);
366 }
367 }
368
369 bool AddressSpaceStream::isInError() const {
370 return 1 == m_context.ring_config->in_error;
371 }
372
373 ssize_t AddressSpaceStream::speculativeRead(unsigned char* readBuffer, size_t trySize) {
374 ensureType3Finished();
375 ensureType1Finished();
376
377 size_t actuallyRead = 0;
378
379 while (!actuallyRead) {
380
381 uint32_t readAvail =
382 ring_buffer_available_read(
383 m_context.from_host_large_xfer.ring,
384 &m_context.from_host_large_xfer.view);
385
386 if (!readAvail) {
387 ring_buffer_yield();
388 backoff();
389 continue;
390 }
391
392 uint32_t toRead = readAvail > trySize ? trySize : readAvail;
393
394 long stepsRead = ring_buffer_view_read(
395 m_context.from_host_large_xfer.ring,
396 &m_context.from_host_large_xfer.view,
397 readBuffer, toRead, 1);
398
399 actuallyRead += stepsRead * toRead;
400
401 if (isInError()) {
402 return -1;
403 }
404 }
405
406 return actuallyRead;
407 }
408
409 void AddressSpaceStream::notifyAvailable() {
410 MESA_TRACE_SCOPE("PING");
411 struct address_space_ping request;
412 request.metadata = ASG_NOTIFY_AVAILABLE;
413 request.resourceId = m_resourceId;
414 m_ops.ping(m_handle, &request);
415 ++m_notifs;
416 }
417
418 uint32_t AddressSpaceStream::getRelativeBufferPos(uint32_t pos) {
419 return pos & m_writeBufferMask;
420 }
421
422 void AddressSpaceStream::advanceWrite() {
423 m_writeStart += m_context.ring_config->flush_interval;
424
425 if (m_writeStart == m_buf + m_context.ring_config->buffer_size) {
426 m_writeStart = m_buf;
427 }
428 }
429
430 void AddressSpaceStream::ensureConsumerFinishing() {
431 uint32_t currAvailRead = ring_buffer_available_read(m_context.to_host, 0);
432
433 while (currAvailRead) {
434 ring_buffer_yield();
435 uint32_t nextAvailRead = ring_buffer_available_read(m_context.to_host, 0);
436
437 if (nextAvailRead != currAvailRead) {
438 break;
439 }
440
441 if (*(m_context.host_state) != ASG_HOST_STATE_CAN_CONSUME &&
442 *(m_context.host_state) != ASG_HOST_STATE_RENDERING) {
443 notifyAvailable();
444 break;
445 }
446
447 backoff();
448 }
449 }
450
451 void AddressSpaceStream::ensureType1Finished() {
452 MESA_TRACE_SCOPE("ensureType1Finished");
453
454 uint32_t currAvailRead =
455 ring_buffer_available_read(m_context.to_host, 0);
456
457 while (currAvailRead) {
458 backoff();
459 ring_buffer_yield();
460 currAvailRead = ring_buffer_available_read(m_context.to_host, 0);
461 if (isInError()) {
462 return;
463 }
464 }
465 }
466
467 void AddressSpaceStream::ensureType3Finished() {
468 MESA_TRACE_SCOPE("ensureType3Finished");
469 uint32_t availReadLarge =
470 ring_buffer_available_read(
471 m_context.to_host_large_xfer.ring,
472 &m_context.to_host_large_xfer.view);
473 while (availReadLarge) {
474 ring_buffer_yield();
475 backoff();
476 availReadLarge =
477 ring_buffer_available_read(
478 m_context.to_host_large_xfer.ring,
479 &m_context.to_host_large_xfer.view);
480 if (*(m_context.host_state) != ASG_HOST_STATE_CAN_CONSUME &&
481 *(m_context.host_state) != ASG_HOST_STATE_RENDERING) {
482 notifyAvailable();
483 }
484 if (isInError()) {
485 return;
486 }
487 }
488 }
489
490 int AddressSpaceStream::type1Write(uint32_t bufferOffset, size_t size) {
491 MESA_TRACE_SCOPE("type1Write");
492
493 ensureType3Finished();
494
495 size_t sent = 0;
496 size_t sizeForRing = sizeof(struct asg_type1_xfer);
497
498 struct asg_type1_xfer xfer = {
499 bufferOffset,
500 (uint32_t)size,
501 };
502
503 uint8_t* writeBufferBytes = (uint8_t*)(&xfer);
504
505 uint32_t maxOutstanding = 1;
506 uint32_t maxSteps = m_context.ring_config->buffer_size /
507 m_context.ring_config->flush_interval;
508
509 if (maxSteps > 1) maxOutstanding = maxSteps - 1;
510
511 uint32_t ringAvailReadNow = ring_buffer_available_read(m_context.to_host, 0);
512
513 while (ringAvailReadNow >= maxOutstanding * sizeForRing) {
514 ringAvailReadNow = ring_buffer_available_read(m_context.to_host, 0);
515 }
516
517 bool hostPinged = false;
518 while (sent < sizeForRing) {
519
520 long sentChunks = ring_buffer_write(
521 m_context.to_host,
522 writeBufferBytes + sent,
523 sizeForRing - sent, 1);
524
525 if (!hostPinged &&
526 *(m_context.host_state) != ASG_HOST_STATE_CAN_CONSUME &&
527 *(m_context.host_state) != ASG_HOST_STATE_RENDERING) {
528 notifyAvailable();
529 hostPinged = true;
530 }
531
532 if (sentChunks == 0) {
533 ring_buffer_yield();
534 backoff();
535 }
536
537 sent += sentChunks * (sizeForRing - sent);
538
539 if (isInError()) {
540 return -1;
541 }
542 }
543
544 bool isRenderingAfter = ASG_HOST_STATE_RENDERING == __atomic_load_n(m_context.host_state, __ATOMIC_ACQUIRE);
545
546 if (!isRenderingAfter) {
547 notifyAvailable();
548 }
549
550 m_written += size;
551
552 float mb = (float)m_written / 1048576.0f;
553 if (mb > 100.0f) {
554 mesa_logd("%s: %f mb in %d notifs. %f mb/notif\n", __func__, mb, m_notifs,
555 m_notifs ? mb / (float)m_notifs : 0.0f);
556 m_notifs = 0;
557 m_written = 0;
558 }
559
560 resetBackoff();
561 return 0;
562 }
563
564 void AddressSpaceStream::backoff() {
565 static const uint32_t kBackoffItersThreshold = 50000000;
566 static const uint32_t kBackoffFactorDoublingIncrement = 50000000;
567 ++m_backoffIters;
568
569 if (m_backoffIters > kBackoffItersThreshold) {
570 usleep(m_backoffFactor);
571 uint32_t itersSoFarAfterThreshold = m_backoffIters - kBackoffItersThreshold;
572 if (itersSoFarAfterThreshold > kBackoffFactorDoublingIncrement) {
573 m_backoffFactor = m_backoffFactor << 1;
574 if (m_backoffFactor > 1000) m_backoffFactor = 1000;
575 m_backoffIters = kBackoffItersThreshold;
576 }
577 }
578 }
579
580 void AddressSpaceStream::resetBackoff() {
581 m_backoffIters = 0;
582 m_backoffFactor = 1;
583 }