| 1 |
use alloc::{boxed::Box, sync::Arc, vec, vec::Vec}; |
| 2 |
use core::{ |
| 3 |
iter, |
| 4 |
mem::{self, ManuallyDrop}, |
| 5 |
ptr::NonNull, |
| 6 |
sync::atomic::Ordering, |
| 7 |
}; |
| 8 |
|
| 9 |
use smallvec::SmallVec; |
| 10 |
use thiserror::Error; |
| 11 |
|
| 12 |
use super::{life::LifetimeTracker, Device}; |
| 13 |
use crate::device::resource::CommandIndices; |
| 14 |
#[cfg(feature = "trace")] |
| 15 |
use crate::device::trace::Action; |
| 16 |
use crate::scratch::ScratchBuffer; |
| 17 |
use crate::{ |
| 18 |
api_log, |
| 19 |
command::{ |
| 20 |
extract_texture_selector, validate_linear_texture_data, validate_texture_copy_range, |
| 21 |
ClearError, CommandAllocator, CommandBuffer, CommandEncoderError, CopySide, |
| 22 |
TexelCopyTextureInfo, TransferError, |
| 23 |
}, |
| 24 |
conv, |
| 25 |
device::{DeviceError, WaitIdleError}, |
| 26 |
get_lowest_common_denom, |
| 27 |
global::Global, |
| 28 |
id::{self, QueueId}, |
| 29 |
init_tracker::{has_copy_partial_init_tracker_coverage, TextureInitRange}, |
| 30 |
lock::{rank, Mutex, MutexGuard, RwLockWriteGuard}, |
| 31 |
resource::{ |
| 32 |
Buffer, BufferAccessError, BufferMapState, DestroyedBuffer, DestroyedResourceError, |
| 33 |
DestroyedTexture, Fallible, FlushedStagingBuffer, InvalidResourceError, Labeled, |
| 34 |
ParentDevice, ResourceErrorIdent, StagingBuffer, Texture, TextureInner, Trackable, |
| 35 |
}, |
| 36 |
resource_log, |
| 37 |
snatch::SnatchGuard, |
| 38 |
track::{self, Tracker, TrackerIndex}, |
| 39 |
FastHashMap, SubmissionIndex, |
| 40 |
}; |
| 41 |
|
| 42 |
pub struct Queue { |
| 43 |
raw: Box<dyn hal::DynQueue>, |
| 44 |
pub(crate) pending_writes: Mutex<PendingWrites>, |
| 45 |
life_tracker: Mutex<LifetimeTracker>, |
| 46 |
// The device needs to be dropped last (`Device.zero_buffer` might be referenced by the encoder in pending writes). |
| 47 |
pub(crate) device: Arc<Device>, |
| 48 |
} |
| 49 |
|
| 50 |
impl Queue { |
| 51 |
pub(crate) fn new( |
| 52 |
device: Arc<Device>, |
| 53 |
raw: Box<dyn hal::DynQueue>, |
| 54 |
) -> Result<Self, DeviceError> { |
| 55 |
let pending_encoder = device |
| 56 |
.command_allocator |
| 57 |
.acquire_encoder(device.raw(), raw.as_ref()) |
| 58 |
.map_err(DeviceError::from_hal); |
| 59 |
|
| 60 |
let pending_encoder = match pending_encoder { |
| 61 |
Ok(pending_encoder) => pending_encoder, |
| 62 |
Err(e) => { |
| 63 |
return Err(e); |
| 64 |
} |
| 65 |
}; |
| 66 |
|
| 67 |
let mut pending_writes = PendingWrites::new(pending_encoder); |
| 68 |
|
| 69 |
let zero_buffer = device.zero_buffer.as_ref(); |
| 70 |
pending_writes.activate(); |
| 71 |
unsafe { |
| 72 |
pending_writes |
| 73 |
.command_encoder |
| 74 |
.transition_buffers(&[hal::BufferBarrier { |
| 75 |
buffer: zero_buffer, |
| 76 |
usage: hal::StateTransition { |
| 77 |
from: wgt::BufferUses::empty(), |
| 78 |
to: wgt::BufferUses::COPY_DST, |
| 79 |
}, |
| 80 |
}]); |
| 81 |
pending_writes |
| 82 |
.command_encoder |
| 83 |
.clear_buffer(zero_buffer, 0..super::ZERO_BUFFER_SIZE); |
| 84 |
pending_writes |
| 85 |
.command_encoder |
| 86 |
.transition_buffers(&[hal::BufferBarrier { |
| 87 |
buffer: zero_buffer, |
| 88 |
usage: hal::StateTransition { |
| 89 |
from: wgt::BufferUses::COPY_DST, |
| 90 |
to: wgt::BufferUses::COPY_SRC, |
| 91 |
}, |
| 92 |
}]); |
| 93 |
} |
| 94 |
|
| 95 |
Ok(Queue { |
| 96 |
raw, |
| 97 |
device, |
| 98 |
pending_writes: Mutex::new(rank::QUEUE_PENDING_WRITES, pending_writes), |
| 99 |
life_tracker: Mutex::new(rank::QUEUE_LIFE_TRACKER, LifetimeTracker::new()), |
| 100 |
}) |
| 101 |
} |
| 102 |
|
| 103 |
pub(crate) fn raw(&self) -> &dyn hal::DynQueue { |
| 104 |
self.raw.as_ref() |
| 105 |
} |
| 106 |
|
| 107 |
#[track_caller] |
| 108 |
pub(crate) fn lock_life<'a>(&'a self) -> MutexGuard<'a, LifetimeTracker> { |
| 109 |
self.life_tracker.lock() |
| 110 |
} |
| 111 |
|
| 112 |
pub(crate) fn maintain( |
| 113 |
&self, |
| 114 |
submission_index: u64, |
| 115 |
snatch_guard: &SnatchGuard, |
| 116 |
) -> ( |
| 117 |
SmallVec<[SubmittedWorkDoneClosure; 1]>, |
| 118 |
Vec<super::BufferMapPendingClosure>, |
| 119 |
bool, |
| 120 |
) { |
| 121 |
let mut life_tracker = self.lock_life(); |
| 122 |
let submission_closures = life_tracker.triage_submissions(submission_index); |
| 123 |
|
| 124 |
let mapping_closures = life_tracker.handle_mapping(snatch_guard); |
| 125 |
|
| 126 |
let queue_empty = life_tracker.queue_empty(); |
| 127 |
|
| 128 |
(submission_closures, mapping_closures, queue_empty) |
| 129 |
} |
| 130 |
} |
| 131 |
|
| 132 |
crate::impl_resource_type!(Queue); |
| 133 |
// TODO: https://github.com/gfx-rs/wgpu/issues/4014 |
| 134 |
impl Labeled for Queue { |
| 135 |
fn label(&self) -> &str { |
| 136 |
"" |
| 137 |
} |
| 138 |
} |
| 139 |
crate::impl_parent_device!(Queue); |
| 140 |
crate::impl_storage_item!(Queue); |
| 141 |
|
| 142 |
impl Drop for Queue { |
| 143 |
fn drop(&mut self) { |
| 144 |
resource_log!("Drop {}", self.error_ident()); |
| 145 |
|
| 146 |
let last_successful_submission_index = self |
| 147 |
.device |
| 148 |
.last_successful_submission_index |
| 149 |
.load(Ordering::Acquire); |
| 150 |
|
| 151 |
let fence = self.device.fence.read(); |
| 152 |
|
| 153 |
// Try waiting on the last submission using the following sequence of timeouts |
| 154 |
let timeouts_in_ms = [100, 200, 400, 800, 1600, 3200]; |
| 155 |
|
| 156 |
for (i, timeout_ms) in timeouts_in_ms.into_iter().enumerate() { |
| 157 |
let is_last_iter = i == timeouts_in_ms.len() - 1; |
| 158 |
|
| 159 |
api_log!( |
| 160 |
"Waiting on last submission. try: {}/{}. timeout: {}ms", |
| 161 |
i + 1, |
| 162 |
timeouts_in_ms.len(), |
| 163 |
timeout_ms |
| 164 |
); |
| 165 |
|
| 166 |
let wait_res = unsafe { |
| 167 |
self.device.raw().wait( |
| 168 |
fence.as_ref(), |
| 169 |
last_successful_submission_index, |
| 170 |
#[cfg(not(target_arch = "wasm32"))] |
| 171 |
timeout_ms, |
| 172 |
#[cfg(target_arch = "wasm32")] |
| 173 |
0, // WebKit and Chromium don't support a non-0 timeout |
| 174 |
) |
| 175 |
}; |
| 176 |
// Note: If we don't panic below we are in UB land (destroying resources while they are still in use by the GPU). |
| 177 |
match wait_res { |
| 178 |
Ok(true) => break, |
| 179 |
Ok(false) => { |
| 180 |
// It's fine that we timed out on WebGL; GL objects can be deleted early as they |
| 181 |
// will be kept around by the driver if GPU work hasn't finished. |
| 182 |
// Moreover, the way we emulate read mappings on WebGL allows us to execute map_buffer earlier than on other |
| 183 |
// backends since getBufferSubData is synchronous with respect to the other previously enqueued GL commands. |
| 184 |
// Relying on this behavior breaks the clean abstraction wgpu-hal tries to maintain and |
| 185 |
// we should find ways to improve this. See https://github.com/gfx-rs/wgpu/issues/6538. |
| 186 |
#[cfg(target_arch = "wasm32")] |
| 187 |
{ |
| 188 |
break; |
| 189 |
} |
| 190 |
#[cfg(not(target_arch = "wasm32"))] |
| 191 |
{ |
| 192 |
if is_last_iter { |
| 193 |
panic!( |
| 194 |
"We timed out while waiting on the last successful submission to complete!" |
| 195 |
); |
| 196 |
} |
| 197 |
} |
| 198 |
} |
| 199 |
Err(e) => match e { |
| 200 |
hal::DeviceError::OutOfMemory => { |
| 201 |
if is_last_iter { |
| 202 |
panic!( |
| 203 |
"We ran into an OOM error while waiting on the last successful submission to complete!" |
| 204 |
); |
| 205 |
} |
| 206 |
} |
| 207 |
hal::DeviceError::Lost => { |
| 208 |
self.device.handle_hal_error(e); // will lose the device |
| 209 |
break; |
| 210 |
} |
| 211 |
hal::DeviceError::ResourceCreationFailed => unreachable!(), |
| 212 |
hal::DeviceError::Unexpected => { |
| 213 |
panic!( |
| 214 |
"We ran into an unexpected error while waiting on the last successful submission to complete!" |
| 215 |
); |
| 216 |
} |
| 217 |
}, |
| 218 |
} |
| 219 |
} |
| 220 |
drop(fence); |
| 221 |
|
| 222 |
let snatch_guard = self.device.snatchable_lock.read(); |
| 223 |
let (submission_closures, mapping_closures, queue_empty) = |
| 224 |
self.maintain(last_successful_submission_index, &snatch_guard); |
| 225 |
drop(snatch_guard); |
| 226 |
|
| 227 |
assert!(queue_empty); |
| 228 |
|
| 229 |
let closures = crate::device::UserClosures { |
| 230 |
mappings: mapping_closures, |
| 231 |
submissions: submission_closures, |
| 232 |
device_lost_invocations: SmallVec::new(), |
| 233 |
}; |
| 234 |
|
| 235 |
closures.fire(); |
| 236 |
} |
| 237 |
} |
| 238 |
|
| 239 |
#[cfg(send_sync)] |
| 240 |
pub type SubmittedWorkDoneClosure = Box<dyn FnOnce() + Send + 'static>; |
| 241 |
#[cfg(not(send_sync))] |
| 242 |
pub type SubmittedWorkDoneClosure = Box<dyn FnOnce() + 'static>; |
| 243 |
|
| 244 |
/// A texture or buffer to be freed soon. |
| 245 |
/// |
| 246 |
/// This is just a tagged raw texture or buffer, generally about to be added to |
| 247 |
/// some other more specific container like: |
| 248 |
/// |
| 249 |
/// - `PendingWrites::temp_resources`: resources used by queue writes and |
| 250 |
/// unmaps, waiting to be folded in with the next queue submission |
| 251 |
/// |
| 252 |
/// - `ActiveSubmission::temp_resources`: temporary resources used by a queue |
| 253 |
/// submission, to be freed when it completes |
| 254 |
#[derive(Debug)] |
| 255 |
pub enum TempResource { |
| 256 |
StagingBuffer(FlushedStagingBuffer), |
| 257 |
ScratchBuffer(ScratchBuffer), |
| 258 |
DestroyedBuffer(DestroyedBuffer), |
| 259 |
DestroyedTexture(DestroyedTexture), |
| 260 |
} |
| 261 |
|
| 262 |
/// A series of raw [`CommandBuffer`]s that have been submitted to a |
| 263 |
/// queue, and the [`wgpu_hal::CommandEncoder`] that built them. |
| 264 |
/// |
| 265 |
/// [`CommandBuffer`]: hal::Api::CommandBuffer |
| 266 |
/// [`wgpu_hal::CommandEncoder`]: hal::CommandEncoder |
| 267 |
pub(crate) struct EncoderInFlight { |
| 268 |
inner: crate::command::CommandEncoder, |
| 269 |
pub(crate) trackers: Tracker, |
| 270 |
pub(crate) temp_resources: Vec<TempResource>, |
| 271 |
/// We only need to keep these resources alive. |
| 272 |
_indirect_draw_validation_resources: crate::indirect_validation::DrawResources, |
| 273 |
|
| 274 |
/// These are the buffers that have been tracked by `PendingWrites`. |
| 275 |
pub(crate) pending_buffers: FastHashMap<TrackerIndex, Arc<Buffer>>, |
| 276 |
/// These are the textures that have been tracked by `PendingWrites`. |
| 277 |
pub(crate) pending_textures: FastHashMap<TrackerIndex, Arc<Texture>>, |
| 278 |
} |
| 279 |
|
| 280 |
/// A private command encoder for writes made directly on the device |
| 281 |
/// or queue. |
| 282 |
/// |
| 283 |
/// Operations like `buffer_unmap`, `queue_write_buffer`, and |
| 284 |
/// `queue_write_texture` need to copy data to the GPU. At the hal |
| 285 |
/// level, this must be done by encoding and submitting commands, but |
| 286 |
/// these operations are not associated with any specific wgpu command |
| 287 |
/// buffer. |
| 288 |
/// |
| 289 |
/// Instead, `Device::pending_writes` owns one of these values, which |
| 290 |
/// has its own hal command encoder and resource lists. The commands |
| 291 |
/// accumulated here are automatically submitted to the queue the next |
| 292 |
/// time the user submits a wgpu command buffer, ahead of the user's |
| 293 |
/// commands. |
| 294 |
/// |
| 295 |
/// Important: |
| 296 |
/// When locking pending_writes be sure that tracker is not locked |
| 297 |
/// and try to lock trackers for the minimum timespan possible |
| 298 |
/// |
| 299 |
/// All uses of [`StagingBuffer`]s end up here. |
| 300 |
#[derive(Debug)] |
| 301 |
pub(crate) struct PendingWrites { |
| 302 |
// The command encoder needs to be destroyed before any other resource in pending writes. |
| 303 |
pub command_encoder: Box<dyn hal::DynCommandEncoder>, |
| 304 |
|
| 305 |
/// True if `command_encoder` is in the "recording" state, as |
| 306 |
/// described in the docs for the [`wgpu_hal::CommandEncoder`] |
| 307 |
/// trait. |
| 308 |
/// |
| 309 |
/// [`wgpu_hal::CommandEncoder`]: hal::CommandEncoder |
| 310 |
pub is_recording: bool, |
| 311 |
|
| 312 |
temp_resources: Vec<TempResource>, |
| 313 |
dst_buffers: FastHashMap<TrackerIndex, Arc<Buffer>>, |
| 314 |
dst_textures: FastHashMap<TrackerIndex, Arc<Texture>>, |
| 315 |
} |
| 316 |
|
| 317 |
impl PendingWrites { |
| 318 |
pub fn new(command_encoder: Box<dyn hal::DynCommandEncoder>) -> Self { |
| 319 |
Self { |
| 320 |
command_encoder, |
| 321 |
is_recording: false, |
| 322 |
temp_resources: Vec::new(), |
| 323 |
dst_buffers: FastHashMap::default(), |
| 324 |
dst_textures: FastHashMap::default(), |
| 325 |
} |
| 326 |
} |
| 327 |
|
| 328 |
pub fn insert_buffer(&mut self, buffer: &Arc<Buffer>) { |
| 329 |
self.dst_buffers |
| 330 |
.insert(buffer.tracker_index(), buffer.clone()); |
| 331 |
} |
| 332 |
|
| 333 |
pub fn insert_texture(&mut self, texture: &Arc<Texture>) { |
| 334 |
self.dst_textures |
| 335 |
.insert(texture.tracker_index(), texture.clone()); |
| 336 |
} |
| 337 |
|
| 338 |
pub fn contains_buffer(&self, buffer: &Arc<Buffer>) -> bool { |
| 339 |
self.dst_buffers.contains_key(&buffer.tracker_index()) |
| 340 |
} |
| 341 |
|
| 342 |
pub fn contains_texture(&self, texture: &Arc<Texture>) -> bool { |
| 343 |
self.dst_textures.contains_key(&texture.tracker_index()) |
| 344 |
} |
| 345 |
|
| 346 |
pub fn consume_temp(&mut self, resource: TempResource) { |
| 347 |
self.temp_resources.push(resource); |
| 348 |
} |
| 349 |
|
| 350 |
pub fn consume(&mut self, buffer: FlushedStagingBuffer) { |
| 351 |
self.temp_resources |
| 352 |
.push(TempResource::StagingBuffer(buffer)); |
| 353 |
} |
| 354 |
|
| 355 |
fn pre_submit( |
| 356 |
&mut self, |
| 357 |
command_allocator: &CommandAllocator, |
| 358 |
device: &Arc<Device>, |
| 359 |
queue: &Queue, |
| 360 |
) -> Result<Option<EncoderInFlight>, DeviceError> { |
| 361 |
if self.is_recording { |
| 362 |
let pending_buffers = mem::take(&mut self.dst_buffers); |
| 363 |
let pending_textures = mem::take(&mut self.dst_textures); |
| 364 |
|
| 365 |
let cmd_buf = unsafe { self.command_encoder.end_encoding() } |
| 366 |
.map_err(|e| device.handle_hal_error(e))?; |
| 367 |
self.is_recording = false; |
| 368 |
|
| 369 |
let new_encoder = command_allocator |
| 370 |
.acquire_encoder(device.raw(), queue.raw()) |
| 371 |
.map_err(|e| device.handle_hal_error(e))?; |
| 372 |
|
| 373 |
let encoder = EncoderInFlight { |
| 374 |
inner: crate::command::CommandEncoder { |
| 375 |
raw: ManuallyDrop::new(mem::replace(&mut self.command_encoder, new_encoder)), |
| 376 |
list: vec![cmd_buf], |
| 377 |
device: device.clone(), |
| 378 |
is_open: false, |
| 379 |
hal_label: None, |
| 380 |
}, |
| 381 |
trackers: Tracker::new(), |
| 382 |
temp_resources: mem::take(&mut self.temp_resources), |
| 383 |
_indirect_draw_validation_resources: crate::indirect_validation::DrawResources::new( |
| 384 |
device.clone(), |
| 385 |
), |
| 386 |
pending_buffers, |
| 387 |
pending_textures, |
| 388 |
}; |
| 389 |
Ok(Some(encoder)) |
| 390 |
} else { |
| 391 |
self.dst_buffers.clear(); |
| 392 |
self.dst_textures.clear(); |
| 393 |
Ok(None) |
| 394 |
} |
| 395 |
} |
| 396 |
|
| 397 |
pub fn activate(&mut self) -> &mut dyn hal::DynCommandEncoder { |
| 398 |
if !self.is_recording { |
| 399 |
unsafe { |
| 400 |
self.command_encoder |
| 401 |
.begin_encoding(Some("(wgpu internal) PendingWrites")) |
| 402 |
.unwrap(); |
| 403 |
} |
| 404 |
self.is_recording = true; |
| 405 |
} |
| 406 |
self.command_encoder.as_mut() |
| 407 |
} |
| 408 |
} |
| 409 |
|
| 410 |
impl Drop for PendingWrites { |
| 411 |
fn drop(&mut self) { |
| 412 |
unsafe { |
| 413 |
if self.is_recording { |
| 414 |
self.command_encoder.discard_encoding(); |
| 415 |
} |
| 416 |
} |
| 417 |
} |
| 418 |
} |
| 419 |
|
| 420 |
#[derive(Clone, Debug, Error)] |
| 421 |
#[non_exhaustive] |
| 422 |
pub enum QueueWriteError { |
| 423 |
#[error(transparent)] |
| 424 |
Queue(#[from] DeviceError), |
| 425 |
#[error(transparent)] |
| 426 |
Transfer(#[from] TransferError), |
| 427 |
#[error(transparent)] |
| 428 |
MemoryInitFailure(#[from] ClearError), |
| 429 |
#[error(transparent)] |
| 430 |
DestroyedResource(#[from] DestroyedResourceError), |
| 431 |
#[error(transparent)] |
| 432 |
InvalidResource(#[from] InvalidResourceError), |
| 433 |
} |
| 434 |
|
| 435 |
#[derive(Clone, Debug, Error)] |
| 436 |
#[non_exhaustive] |
| 437 |
pub enum QueueSubmitError { |
| 438 |
#[error(transparent)] |
| 439 |
Queue(#[from] DeviceError), |
| 440 |
#[error(transparent)] |
| 441 |
DestroyedResource(#[from] DestroyedResourceError), |
| 442 |
#[error(transparent)] |
| 443 |
Unmap(#[from] BufferAccessError), |
| 444 |
#[error("{0} is still mapped")] |
| 445 |
BufferStillMapped(ResourceErrorIdent), |
| 446 |
#[error(transparent)] |
| 447 |
InvalidResource(#[from] InvalidResourceError), |
| 448 |
#[error(transparent)] |
| 449 |
CommandEncoder(#[from] CommandEncoderError), |
| 450 |
#[error(transparent)] |
| 451 |
ValidateAsActionsError(#[from] crate::ray_tracing::ValidateAsActionsError), |
| 452 |
} |
| 453 |
|
| 454 |
//TODO: move out common parts of write_xxx. |
| 455 |
|
| 456 |
impl Queue { |
| 457 |
pub fn write_buffer( |
| 458 |
&self, |
| 459 |
buffer: Fallible<Buffer>, |
| 460 |
buffer_offset: wgt::BufferAddress, |
| 461 |
data: &[u8], |
| 462 |
) -> Result<(), QueueWriteError> { |
| 463 |
profiling::scope!("Queue::write_buffer"); |
| 464 |
api_log!("Queue::write_buffer"); |
| 465 |
|
| 466 |
self.device.check_is_valid()?; |
| 467 |
|
| 468 |
let buffer = buffer.get()?; |
| 469 |
|
| 470 |
let data_size = data.len() as wgt::BufferAddress; |
| 471 |
|
| 472 |
self.same_device_as(buffer.as_ref())?; |
| 473 |
|
| 474 |
let data_size = if let Some(data_size) = wgt::BufferSize::new(data_size) { |
| 475 |
data_size |
| 476 |
} else { |
| 477 |
log::trace!("Ignoring write_buffer of size 0"); |
| 478 |
return Ok(()); |
| 479 |
}; |
| 480 |
|
| 481 |
let snatch_guard = self.device.snatchable_lock.read(); |
| 482 |
|
| 483 |
// Platform validation requires that the staging buffer always be |
| 484 |
// freed, even if an error occurs. All paths from here must call |
| 485 |
// `device.pending_writes.consume`. |
| 486 |
let mut staging_buffer = StagingBuffer::new(&self.device, data_size)?; |
| 487 |
let mut pending_writes = self.pending_writes.lock(); |
| 488 |
|
| 489 |
let staging_buffer = { |
| 490 |
profiling::scope!("copy"); |
| 491 |
staging_buffer.write(data); |
| 492 |
staging_buffer.flush() |
| 493 |
}; |
| 494 |
|
| 495 |
let result = self.write_staging_buffer_impl( |
| 496 |
&snatch_guard, |
| 497 |
&mut pending_writes, |
| 498 |
&staging_buffer, |
| 499 |
buffer, |
| 500 |
buffer_offset, |
| 501 |
); |
| 502 |
|
| 503 |
pending_writes.consume(staging_buffer); |
| 504 |
result |
| 505 |
} |
| 506 |
|
| 507 |
pub fn create_staging_buffer( |
| 508 |
&self, |
| 509 |
buffer_size: wgt::BufferSize, |
| 510 |
) -> Result<(StagingBuffer, NonNull<u8>), QueueWriteError> { |
| 511 |
profiling::scope!("Queue::create_staging_buffer"); |
| 512 |
resource_log!("Queue::create_staging_buffer"); |
| 513 |
|
| 514 |
self.device.check_is_valid()?; |
| 515 |
|
| 516 |
let staging_buffer = StagingBuffer::new(&self.device, buffer_size)?; |
| 517 |
let ptr = unsafe { staging_buffer.ptr() }; |
| 518 |
|
| 519 |
Ok((staging_buffer, ptr)) |
| 520 |
} |
| 521 |
|
| 522 |
pub fn write_staging_buffer( |
| 523 |
&self, |
| 524 |
buffer: Fallible<Buffer>, |
| 525 |
buffer_offset: wgt::BufferAddress, |
| 526 |
staging_buffer: StagingBuffer, |
| 527 |
) -> Result<(), QueueWriteError> { |
| 528 |
profiling::scope!("Queue::write_staging_buffer"); |
| 529 |
|
| 530 |
self.device.check_is_valid()?; |
| 531 |
|
| 532 |
let buffer = buffer.get()?; |
| 533 |
|
| 534 |
let snatch_guard = self.device.snatchable_lock.read(); |
| 535 |
let mut pending_writes = self.pending_writes.lock(); |
| 536 |
|
| 537 |
// At this point, we have taken ownership of the staging_buffer from the |
| 538 |
// user. Platform validation requires that the staging buffer always |
| 539 |
// be freed, even if an error occurs. All paths from here must call |
| 540 |
// `device.pending_writes.consume`. |
| 541 |
let staging_buffer = staging_buffer.flush(); |
| 542 |
|
| 543 |
let result = self.write_staging_buffer_impl( |
| 544 |
&snatch_guard, |
| 545 |
&mut pending_writes, |
| 546 |
&staging_buffer, |
| 547 |
buffer, |
| 548 |
buffer_offset, |
| 549 |
); |
| 550 |
|
| 551 |
pending_writes.consume(staging_buffer); |
| 552 |
result |
| 553 |
} |
| 554 |
|
| 555 |
pub fn validate_write_buffer( |
| 556 |
&self, |
| 557 |
buffer: Fallible<Buffer>, |
| 558 |
buffer_offset: u64, |
| 559 |
buffer_size: wgt::BufferSize, |
| 560 |
) -> Result<(), QueueWriteError> { |
| 561 |
profiling::scope!("Queue::validate_write_buffer"); |
| 562 |
|
| 563 |
self.device.check_is_valid()?; |
| 564 |
|
| 565 |
let buffer = buffer.get()?; |
| 566 |
|
| 567 |
self.validate_write_buffer_impl(&buffer, buffer_offset, buffer_size)?; |
| 568 |
|
| 569 |
Ok(()) |
| 570 |
} |
| 571 |
|
| 572 |
fn validate_write_buffer_impl( |
| 573 |
&self, |
| 574 |
buffer: &Buffer, |
| 575 |
buffer_offset: u64, |
| 576 |
buffer_size: wgt::BufferSize, |
| 577 |
) -> Result<(), TransferError> { |
| 578 |
buffer.check_usage(wgt::BufferUsages::COPY_DST)?; |
| 579 |
if buffer_size.get() % wgt::COPY_BUFFER_ALIGNMENT != 0 { |
| 580 |
return Err(TransferError::UnalignedCopySize(buffer_size.get())); |
| 581 |
} |
| 582 |
if buffer_offset % wgt::COPY_BUFFER_ALIGNMENT != 0 { |
| 583 |
return Err(TransferError::UnalignedBufferOffset(buffer_offset)); |
| 584 |
} |
| 585 |
if buffer_offset + buffer_size.get() > buffer.size { |
| 586 |
return Err(TransferError::BufferOverrun { |
| 587 |
start_offset: buffer_offset, |
| 588 |
end_offset: buffer_offset + buffer_size.get(), |
| 589 |
buffer_size: buffer.size, |
| 590 |
side: CopySide::Destination, |
| 591 |
}); |
| 592 |
} |
| 593 |
|
| 594 |
Ok(()) |
| 595 |
} |
| 596 |
|
| 597 |
fn write_staging_buffer_impl( |
| 598 |
&self, |
| 599 |
snatch_guard: &SnatchGuard, |
| 600 |
pending_writes: &mut PendingWrites, |
| 601 |
staging_buffer: &FlushedStagingBuffer, |
| 602 |
buffer: Arc<Buffer>, |
| 603 |
buffer_offset: u64, |
| 604 |
) -> Result<(), QueueWriteError> { |
| 605 |
self.device.check_is_valid()?; |
| 606 |
|
| 607 |
let transition = { |
| 608 |
let mut trackers = self.device.trackers.lock(); |
| 609 |
trackers |
| 610 |
.buffers |
| 611 |
.set_single(&buffer, wgt::BufferUses::COPY_DST) |
| 612 |
}; |
| 613 |
|
| 614 |
let dst_raw = buffer.try_raw(snatch_guard)?; |
| 615 |
|
| 616 |
self.same_device_as(buffer.as_ref())?; |
| 617 |
|
| 618 |
self.validate_write_buffer_impl(&buffer, buffer_offset, staging_buffer.size)?; |
| 619 |
|
| 620 |
let region = hal::BufferCopy { |
| 621 |
src_offset: 0, |
| 622 |
dst_offset: buffer_offset, |
| 623 |
size: staging_buffer.size, |
| 624 |
}; |
| 625 |
let barriers = iter::once(hal::BufferBarrier { |
| 626 |
buffer: staging_buffer.raw(), |
| 627 |
usage: hal::StateTransition { |
| 628 |
from: wgt::BufferUses::MAP_WRITE, |
| 629 |
to: wgt::BufferUses::COPY_SRC, |
| 630 |
}, |
| 631 |
}) |
| 632 |
.chain(transition.map(|pending| pending.into_hal(&buffer, snatch_guard))) |
| 633 |
.collect::<Vec<_>>(); |
| 634 |
let encoder = pending_writes.activate(); |
| 635 |
unsafe { |
| 636 |
encoder.transition_buffers(&barriers); |
| 637 |
encoder.copy_buffer_to_buffer(staging_buffer.raw(), dst_raw, &[region]); |
| 638 |
} |
| 639 |
|
| 640 |
pending_writes.insert_buffer(&buffer); |
| 641 |
|
| 642 |
// Ensure the overwritten bytes are marked as initialized so |
| 643 |
// they don't need to be nulled prior to mapping or binding. |
| 644 |
{ |
| 645 |
buffer |
| 646 |
.initialization_status |
| 647 |
.write() |
| 648 |
.drain(buffer_offset..(buffer_offset + staging_buffer.size.get())); |
| 649 |
} |
| 650 |
|
| 651 |
Ok(()) |
| 652 |
} |
| 653 |
|
| 654 |
pub fn write_texture( |
| 655 |
&self, |
| 656 |
destination: wgt::TexelCopyTextureInfo<Fallible<Texture>>, |
| 657 |
data: &[u8], |
| 658 |
data_layout: &wgt::TexelCopyBufferLayout, |
| 659 |
size: &wgt::Extent3d, |
| 660 |
) -> Result<(), QueueWriteError> { |
| 661 |
profiling::scope!("Queue::write_texture"); |
| 662 |
api_log!("Queue::write_texture"); |
| 663 |
|
| 664 |
self.device.check_is_valid()?; |
| 665 |
|
| 666 |
if size.width == 0 || size.height == 0 || size.depth_or_array_layers == 0 { |
| 667 |
log::trace!("Ignoring write_texture of size 0"); |
| 668 |
return Ok(()); |
| 669 |
} |
| 670 |
|
| 671 |
let dst = destination.texture.get()?; |
| 672 |
let destination = wgt::TexelCopyTextureInfo { |
| 673 |
texture: (), |
| 674 |
mip_level: destination.mip_level, |
| 675 |
origin: destination.origin, |
| 676 |
aspect: destination.aspect, |
| 677 |
}; |
| 678 |
|
| 679 |
self.same_device_as(dst.as_ref())?; |
| 680 |
|
| 681 |
dst.check_usage(wgt::TextureUsages::COPY_DST) |
| 682 |
.map_err(TransferError::MissingTextureUsage)?; |
| 683 |
|
| 684 |
// Note: Doing the copy range validation early is important because ensures that the |
| 685 |
// dimensions are not going to cause overflow in other parts of the validation. |
| 686 |
let (hal_copy_size, array_layer_count) = |
| 687 |
validate_texture_copy_range(&destination, &dst.desc, CopySide::Destination, size)?; |
| 688 |
|
| 689 |
let (selector, dst_base) = extract_texture_selector(&destination, size, &dst)?; |
| 690 |
|
| 691 |
if !dst_base.aspect.is_one() { |
| 692 |
return Err(TransferError::CopyAspectNotOne.into()); |
| 693 |
} |
| 694 |
|
| 695 |
if !conv::is_valid_copy_dst_texture_format(dst.desc.format, destination.aspect) { |
| 696 |
return Err(TransferError::CopyToForbiddenTextureFormat { |
| 697 |
format: dst.desc.format, |
| 698 |
aspect: destination.aspect, |
| 699 |
} |
| 700 |
.into()); |
| 701 |
} |
| 702 |
|
| 703 |
// Note: `_source_bytes_per_array_layer` is ignored since we |
| 704 |
// have a staging copy, and it can have a different value. |
| 705 |
let (required_bytes_in_copy, _source_bytes_per_array_layer) = validate_linear_texture_data( |
| 706 |
data_layout, |
| 707 |
dst.desc.format, |
| 708 |
destination.aspect, |
| 709 |
data.len() as wgt::BufferAddress, |
| 710 |
CopySide::Source, |
| 711 |
size, |
| 712 |
false, |
| 713 |
)?; |
| 714 |
|
| 715 |
if dst.desc.format.is_depth_stencil_format() { |
| 716 |
self.device |
| 717 |
.require_downlevel_flags(wgt::DownlevelFlags::DEPTH_TEXTURE_AND_BUFFER_COPIES) |
| 718 |
.map_err(TransferError::from)?; |
| 719 |
} |
| 720 |
|
| 721 |
let snatch_guard = self.device.snatchable_lock.read(); |
| 722 |
|
| 723 |
let mut pending_writes = self.pending_writes.lock(); |
| 724 |
let encoder = pending_writes.activate(); |
| 725 |
|
| 726 |
// If the copy does not fully cover the layers, we need to initialize to |
| 727 |
// zero *first* as we don't keep track of partial texture layer inits. |
| 728 |
// |
| 729 |
// Strictly speaking we only need to clear the areas of a layer |
| 730 |
// untouched, but this would get increasingly messy. |
| 731 |
let init_layer_range = if dst.desc.dimension == wgt::TextureDimension::D3 { |
| 732 |
// volume textures don't have a layer range as array volumes aren't supported |
| 733 |
0..1 |
| 734 |
} else { |
| 735 |
destination.origin.z..destination.origin.z + size.depth_or_array_layers |
| 736 |
}; |
| 737 |
let mut dst_initialization_status = dst.initialization_status.write(); |
| 738 |
if dst_initialization_status.mips[destination.mip_level as usize] |
| 739 |
.check(init_layer_range.clone()) |
| 740 |
.is_some() |
| 741 |
{ |
| 742 |
if has_copy_partial_init_tracker_coverage(size, destination.mip_level, &dst.desc) { |
| 743 |
for layer_range in dst_initialization_status.mips[destination.mip_level as usize] |
| 744 |
.drain(init_layer_range) |
| 745 |
.collect::<Vec<core::ops::Range<u32>>>() |
| 746 |
{ |
| 747 |
let mut trackers = self.device.trackers.lock(); |
| 748 |
crate::command::clear_texture( |
| 749 |
&dst, |
| 750 |
TextureInitRange { |
| 751 |
mip_range: destination.mip_level..(destination.mip_level + 1), |
| 752 |
layer_range, |
| 753 |
}, |
| 754 |
encoder, |
| 755 |
&mut trackers.textures, |
| 756 |
&self.device.alignments, |
| 757 |
self.device.zero_buffer.as_ref(), |
| 758 |
&snatch_guard, |
| 759 |
) |
| 760 |
.map_err(QueueWriteError::from)?; |
| 761 |
} |
| 762 |
} else { |
| 763 |
dst_initialization_status.mips[destination.mip_level as usize] |
| 764 |
.drain(init_layer_range); |
| 765 |
} |
| 766 |
} |
| 767 |
|
| 768 |
let dst_raw = dst.try_raw(&snatch_guard)?; |
| 769 |
|
| 770 |
let (block_width, block_height) = dst.desc.format.block_dimensions(); |
| 771 |
let width_in_blocks = size.width / block_width; |
| 772 |
let height_in_blocks = size.height / block_height; |
| 773 |
|
| 774 |
let block_size = dst |
| 775 |
.desc |
| 776 |
.format |
| 777 |
.block_copy_size(Some(destination.aspect)) |
| 778 |
.unwrap(); |
| 779 |
let bytes_in_last_row = width_in_blocks * block_size; |
| 780 |
|
| 781 |
let bytes_per_row = data_layout.bytes_per_row.unwrap_or(bytes_in_last_row); |
| 782 |
let rows_per_image = data_layout.rows_per_image.unwrap_or(height_in_blocks); |
| 783 |
|
| 784 |
let bytes_per_row_alignment = get_lowest_common_denom( |
| 785 |
self.device.alignments.buffer_copy_pitch.get() as u32, |
| 786 |
block_size, |
| 787 |
); |
| 788 |
let stage_bytes_per_row = wgt::math::align_to(bytes_in_last_row, bytes_per_row_alignment); |
| 789 |
|
| 790 |
// Platform validation requires that the staging buffer always be |
| 791 |
// freed, even if an error occurs. All paths from here must call |
| 792 |
// `device.pending_writes.consume`. |
| 793 |
let staging_buffer = if stage_bytes_per_row == bytes_per_row { |
| 794 |
profiling::scope!("copy aligned"); |
| 795 |
// Fast path if the data is already being aligned optimally. |
| 796 |
let stage_size = wgt::BufferSize::new(required_bytes_in_copy).unwrap(); |
| 797 |
let mut staging_buffer = StagingBuffer::new(&self.device, stage_size)?; |
| 798 |
staging_buffer.write(&data[data_layout.offset as usize..]); |
| 799 |
staging_buffer |
| 800 |
} else { |
| 801 |
profiling::scope!("copy chunked"); |
| 802 |
// Copy row by row into the optimal alignment. |
| 803 |
let block_rows_in_copy = |
| 804 |
(size.depth_or_array_layers - 1) * rows_per_image + height_in_blocks; |
| 805 |
let stage_size = |
| 806 |
wgt::BufferSize::new(stage_bytes_per_row as u64 * block_rows_in_copy as u64) |
| 807 |
.unwrap(); |
| 808 |
let mut staging_buffer = StagingBuffer::new(&self.device, stage_size)?; |
| 809 |
let copy_bytes_per_row = stage_bytes_per_row.min(bytes_per_row) as usize; |
| 810 |
for layer in 0..size.depth_or_array_layers { |
| 811 |
let rows_offset = layer * rows_per_image; |
| 812 |
for row in rows_offset..rows_offset + height_in_blocks { |
| 813 |
let src_offset = data_layout.offset as u32 + row * bytes_per_row; |
| 814 |
let dst_offset = row * stage_bytes_per_row; |
| 815 |
unsafe { |
| 816 |
staging_buffer.write_with_offset( |
| 817 |
data, |
| 818 |
src_offset as isize, |
| 819 |
dst_offset as isize, |
| 820 |
copy_bytes_per_row, |
| 821 |
) |
| 822 |
} |
| 823 |
} |
| 824 |
} |
| 825 |
staging_buffer |
| 826 |
}; |
| 827 |
|
| 828 |
let staging_buffer = staging_buffer.flush(); |
| 829 |
|
| 830 |
let regions = (0..array_layer_count) |
| 831 |
.map(|array_layer_offset| { |
| 832 |
let mut texture_base = dst_base.clone(); |
| 833 |
texture_base.array_layer += array_layer_offset; |
| 834 |
hal::BufferTextureCopy { |
| 835 |
buffer_layout: wgt::TexelCopyBufferLayout { |
| 836 |
offset: array_layer_offset as u64 |
| 837 |
* rows_per_image as u64 |
| 838 |
* stage_bytes_per_row as u64, |
| 839 |
bytes_per_row: Some(stage_bytes_per_row), |
| 840 |
rows_per_image: Some(rows_per_image), |
| 841 |
}, |
| 842 |
texture_base, |
| 843 |
size: hal_copy_size, |
| 844 |
} |
| 845 |
}) |
| 846 |
.collect::<Vec<_>>(); |
| 847 |
|
| 848 |
{ |
| 849 |
let buffer_barrier = hal::BufferBarrier { |
| 850 |
buffer: staging_buffer.raw(), |
| 851 |
usage: hal::StateTransition { |
| 852 |
from: wgt::BufferUses::MAP_WRITE, |
| 853 |
to: wgt::BufferUses::COPY_SRC, |
| 854 |
}, |
| 855 |
}; |
| 856 |
|
| 857 |
let mut trackers = self.device.trackers.lock(); |
| 858 |
let transition = |
| 859 |
trackers |
| 860 |
.textures |
| 861 |
.set_single(&dst, selector, wgt::TextureUses::COPY_DST); |
| 862 |
let texture_barriers = transition |
| 863 |
.map(|pending| pending.into_hal(dst_raw)) |
| 864 |
.collect::<Vec<_>>(); |
| 865 |
|
| 866 |
unsafe { |
| 867 |
encoder.transition_textures(&texture_barriers); |
| 868 |
encoder.transition_buffers(&[buffer_barrier]); |
| 869 |
encoder.copy_buffer_to_texture(staging_buffer.raw(), dst_raw, ®ions); |
| 870 |
} |
| 871 |
} |
| 872 |
|
| 873 |
pending_writes.consume(staging_buffer); |
| 874 |
pending_writes.insert_texture(&dst); |
| 875 |
|
| 876 |
Ok(()) |
| 877 |
} |
| 878 |
|
| 879 |
#[cfg(webgl)] |
| 880 |
pub fn copy_external_image_to_texture( |
| 881 |
&self, |
| 882 |
source: &wgt::CopyExternalImageSourceInfo, |
| 883 |
destination: wgt::CopyExternalImageDestInfo<Fallible<Texture>>, |
| 884 |
size: wgt::Extent3d, |
| 885 |
) -> Result<(), QueueWriteError> { |
| 886 |
profiling::scope!("Queue::copy_external_image_to_texture"); |
| 887 |
|
| 888 |
self.device.check_is_valid()?; |
| 889 |
|
| 890 |
if size.width == 0 || size.height == 0 || size.depth_or_array_layers == 0 { |
| 891 |
log::trace!("Ignoring write_texture of size 0"); |
| 892 |
return Ok(()); |
| 893 |
} |
| 894 |
|
| 895 |
let mut needs_flag = false; |
| 896 |
needs_flag |= matches!(source.source, wgt::ExternalImageSource::OffscreenCanvas(_)); |
| 897 |
needs_flag |= source.origin != wgt::Origin2d::ZERO; |
| 898 |
needs_flag |= destination.color_space != wgt::PredefinedColorSpace::Srgb; |
| 899 |
#[allow(clippy::bool_comparison)] |
| 900 |
if matches!(source.source, wgt::ExternalImageSource::ImageBitmap(_)) { |
| 901 |
needs_flag |= source.flip_y != false; |
| 902 |
needs_flag |= destination.premultiplied_alpha != false; |
| 903 |
} |
| 904 |
|
| 905 |
if needs_flag { |
| 906 |
self.device |
| 907 |
.require_downlevel_flags(wgt::DownlevelFlags::UNRESTRICTED_EXTERNAL_TEXTURE_COPIES) |
| 908 |
.map_err(TransferError::from)?; |
| 909 |
} |
| 910 |
|
| 911 |
let src_width = source.source.width(); |
| 912 |
let src_height = source.source.height(); |
| 913 |
|
| 914 |
let dst = destination.texture.get()?; |
| 915 |
let premultiplied_alpha = destination.premultiplied_alpha; |
| 916 |
let destination = wgt::TexelCopyTextureInfo { |
| 917 |
texture: (), |
| 918 |
mip_level: destination.mip_level, |
| 919 |
origin: destination.origin, |
| 920 |
aspect: destination.aspect, |
| 921 |
}; |
| 922 |
|
| 923 |
if !conv::is_valid_external_image_copy_dst_texture_format(dst.desc.format) { |
| 924 |
return Err( |
| 925 |
TransferError::ExternalCopyToForbiddenTextureFormat(dst.desc.format).into(), |
| 926 |
); |
| 927 |
} |
| 928 |
if dst.desc.dimension != wgt::TextureDimension::D2 { |
| 929 |
return Err(TransferError::InvalidDimensionExternal.into()); |
| 930 |
} |
| 931 |
dst.check_usage(wgt::TextureUsages::COPY_DST | wgt::TextureUsages::RENDER_ATTACHMENT) |
| 932 |
.map_err(TransferError::MissingTextureUsage)?; |
| 933 |
if dst.desc.sample_count != 1 { |
| 934 |
return Err(TransferError::InvalidSampleCount { |
| 935 |
sample_count: dst.desc.sample_count, |
| 936 |
} |
| 937 |
.into()); |
| 938 |
} |
| 939 |
|
| 940 |
if source.origin.x + size.width > src_width { |
| 941 |
return Err(TransferError::TextureOverrun { |
| 942 |
start_offset: source.origin.x, |
| 943 |
end_offset: source.origin.x + size.width, |
| 944 |
texture_size: src_width, |
| 945 |
dimension: crate::resource::TextureErrorDimension::X, |
| 946 |
side: CopySide::Source, |
| 947 |
} |
| 948 |
.into()); |
| 949 |
} |
| 950 |
if source.origin.y + size.height > src_height { |
| 951 |
return Err(TransferError::TextureOverrun { |
| 952 |
start_offset: source.origin.y, |
| 953 |
end_offset: source.origin.y + size.height, |
| 954 |
texture_size: src_height, |
| 955 |
dimension: crate::resource::TextureErrorDimension::Y, |
| 956 |
side: CopySide::Source, |
| 957 |
} |
| 958 |
.into()); |
| 959 |
} |
| 960 |
if size.depth_or_array_layers != 1 { |
| 961 |
return Err(TransferError::TextureOverrun { |
| 962 |
start_offset: 0, |
| 963 |
end_offset: size.depth_or_array_layers, |
| 964 |
texture_size: 1, |
| 965 |
dimension: crate::resource::TextureErrorDimension::Z, |
| 966 |
side: CopySide::Source, |
| 967 |
} |
| 968 |
.into()); |
| 969 |
} |
| 970 |
|
| 971 |
// Note: Doing the copy range validation early is important because ensures that the |
| 972 |
// dimensions are not going to cause overflow in other parts of the validation. |
| 973 |
let (hal_copy_size, _) = |
| 974 |
validate_texture_copy_range(&destination, &dst.desc, CopySide::Destination, &size)?; |
| 975 |
|
| 976 |
let (selector, dst_base) = extract_texture_selector(&destination, &size, &dst)?; |
| 977 |
|
| 978 |
let mut pending_writes = self.pending_writes.lock(); |
| 979 |
let encoder = pending_writes.activate(); |
| 980 |
|
| 981 |
// If the copy does not fully cover the layers, we need to initialize to |
| 982 |
// zero *first* as we don't keep track of partial texture layer inits. |
| 983 |
// |
| 984 |
// Strictly speaking we only need to clear the areas of a layer |
| 985 |
// untouched, but this would get increasingly messy. |
| 986 |
let init_layer_range = if dst.desc.dimension == wgt::TextureDimension::D3 { |
| 987 |
// volume textures don't have a layer range as array volumes aren't supported |
| 988 |
0..1 |
| 989 |
} else { |
| 990 |
destination.origin.z..destination.origin.z + size.depth_or_array_layers |
| 991 |
}; |
| 992 |
let mut dst_initialization_status = dst.initialization_status.write(); |
| 993 |
if dst_initialization_status.mips[destination.mip_level as usize] |
| 994 |
.check(init_layer_range.clone()) |
| 995 |
.is_some() |
| 996 |
{ |
| 997 |
if has_copy_partial_init_tracker_coverage(&size, destination.mip_level, &dst.desc) { |
| 998 |
for layer_range in dst_initialization_status.mips[destination.mip_level as usize] |
| 999 |
.drain(init_layer_range) |
| 1000 |
.collect::<Vec<core::ops::Range<u32>>>() |
| 1001 |
{ |
| 1002 |
let mut trackers = self.device.trackers.lock(); |
| 1003 |
crate::command::clear_texture( |
| 1004 |
&dst, |
| 1005 |
TextureInitRange { |
| 1006 |
mip_range: destination.mip_level..(destination.mip_level + 1), |
| 1007 |
layer_range, |
| 1008 |
}, |
| 1009 |
encoder, |
| 1010 |
&mut trackers.textures, |
| 1011 |
&self.device.alignments, |
| 1012 |
self.device.zero_buffer.as_ref(), |
| 1013 |
&self.device.snatchable_lock.read(), |
| 1014 |
) |
| 1015 |
.map_err(QueueWriteError::from)?; |
| 1016 |
} |
| 1017 |
} else { |
| 1018 |
dst_initialization_status.mips[destination.mip_level as usize] |
| 1019 |
.drain(init_layer_range); |
| 1020 |
} |
| 1021 |
} |
| 1022 |
|
| 1023 |
let snatch_guard = self.device.snatchable_lock.read(); |
| 1024 |
let dst_raw = dst.try_raw(&snatch_guard)?; |
| 1025 |
|
| 1026 |
let regions = hal::TextureCopy { |
| 1027 |
src_base: hal::TextureCopyBase { |
| 1028 |
mip_level: 0, |
| 1029 |
array_layer: 0, |
| 1030 |
origin: source.origin.to_3d(0), |
| 1031 |
aspect: hal::FormatAspects::COLOR, |
| 1032 |
}, |
| 1033 |
dst_base, |
| 1034 |
size: hal_copy_size, |
| 1035 |
}; |
| 1036 |
|
| 1037 |
let mut trackers = self.device.trackers.lock(); |
| 1038 |
let transitions = trackers |
| 1039 |
.textures |
| 1040 |
.set_single(&dst, selector, wgt::TextureUses::COPY_DST); |
| 1041 |
|
| 1042 |
// `copy_external_image_to_texture` is exclusive to the WebGL backend. |
| 1043 |
// Don't go through the `DynCommandEncoder` abstraction and directly to the WebGL backend. |
| 1044 |
let encoder_webgl = encoder |
| 1045 |
.as_any_mut() |
| 1046 |
.downcast_mut::<hal::gles::CommandEncoder>() |
| 1047 |
.unwrap(); |
| 1048 |
let dst_raw_webgl = dst_raw |
| 1049 |
.as_any() |
| 1050 |
.downcast_ref::<hal::gles::Texture>() |
| 1051 |
.unwrap(); |
| 1052 |
let transitions_webgl = transitions.map(|pending| { |
| 1053 |
let dyn_transition = pending.into_hal(dst_raw); |
| 1054 |
hal::TextureBarrier { |
| 1055 |
texture: dst_raw_webgl, |
| 1056 |
range: dyn_transition.range, |
| 1057 |
usage: dyn_transition.usage, |
| 1058 |
} |
| 1059 |
}); |
| 1060 |
|
| 1061 |
use hal::CommandEncoder as _; |
| 1062 |
unsafe { |
| 1063 |
encoder_webgl.transition_textures(transitions_webgl); |
| 1064 |
encoder_webgl.copy_external_image_to_texture( |
| 1065 |
source, |
| 1066 |
dst_raw_webgl, |
| 1067 |
premultiplied_alpha, |
| 1068 |
iter::once(regions), |
| 1069 |
); |
| 1070 |
} |
| 1071 |
|
| 1072 |
Ok(()) |
| 1073 |
} |
| 1074 |
|
| 1075 |
pub fn submit( |
| 1076 |
&self, |
| 1077 |
command_buffers: &[Arc<CommandBuffer>], |
| 1078 |
) -> Result<SubmissionIndex, (SubmissionIndex, QueueSubmitError)> { |
| 1079 |
profiling::scope!("Queue::submit"); |
| 1080 |
api_log!("Queue::submit"); |
| 1081 |
|
| 1082 |
let submit_index; |
| 1083 |
|
| 1084 |
let res = 'error: { |
| 1085 |
let snatch_guard = self.device.snatchable_lock.read(); |
| 1086 |
|
| 1087 |
// Fence lock must be acquired after the snatch lock everywhere to avoid deadlocks. |
| 1088 |
let mut fence = self.device.fence.write(); |
| 1089 |
|
| 1090 |
let mut command_index_guard = self.device.command_indices.write(); |
| 1091 |
command_index_guard.active_submission_index += 1; |
| 1092 |
submit_index = command_index_guard.active_submission_index; |
| 1093 |
|
| 1094 |
if let Err(e) = self.device.check_is_valid() { |
| 1095 |
break 'error Err(e.into()); |
| 1096 |
} |
| 1097 |
|
| 1098 |
let mut active_executions = Vec::new(); |
| 1099 |
|
| 1100 |
let mut used_surface_textures = track::TextureUsageScope::default(); |
| 1101 |
|
| 1102 |
// Use a hashmap here to deduplicate the surface textures that are used in the command buffers. |
| 1103 |
// This avoids vulkan deadlocking from the same surface texture being submitted multiple times. |
| 1104 |
let mut submit_surface_textures_owned = FastHashMap::default(); |
| 1105 |
|
| 1106 |
{ |
| 1107 |
if !command_buffers.is_empty() { |
| 1108 |
profiling::scope!("prepare"); |
| 1109 |
|
| 1110 |
let mut first_error = None; |
| 1111 |
|
| 1112 |
//TODO: if multiple command buffers are submitted, we can re-use the last |
| 1113 |
// native command buffer of the previous chain instead of always creating |
| 1114 |
// a temporary one, since the chains are not finished. |
| 1115 |
|
| 1116 |
// finish all the command buffers first |
| 1117 |
for command_buffer in command_buffers { |
| 1118 |
profiling::scope!("process command buffer"); |
| 1119 |
|
| 1120 |
// we reset the used surface textures every time we use |
| 1121 |
// it, so make sure to set_size on it. |
| 1122 |
used_surface_textures.set_size(self.device.tracker_indices.textures.size()); |
| 1123 |
|
| 1124 |
// Note that we are required to invalidate all command buffers in both the success and failure paths. |
| 1125 |
// This is why we `continue` and don't early return via `?`. |
| 1126 |
#[allow(unused_mut)] |
| 1127 |
let mut cmd_buf_data = command_buffer.take_finished(); |
| 1128 |
|
| 1129 |
#[cfg(feature = "trace")] |
| 1130 |
if let Some(ref mut trace) = *self.device.trace.lock() { |
| 1131 |
if let Ok(ref mut cmd_buf_data) = cmd_buf_data { |
| 1132 |
trace.add(Action::Submit( |
| 1133 |
submit_index, |
| 1134 |
cmd_buf_data.commands.take().unwrap(), |
| 1135 |
)); |
| 1136 |
} |
| 1137 |
} |
| 1138 |
|
| 1139 |
if first_error.is_some() { |
| 1140 |
continue; |
| 1141 |
} |
| 1142 |
|
| 1143 |
let mut baked = match cmd_buf_data { |
| 1144 |
Ok(cmd_buf_data) => { |
| 1145 |
let res = validate_command_buffer( |
| 1146 |
command_buffer, |
| 1147 |
self, |
| 1148 |
&cmd_buf_data, |
| 1149 |
&snatch_guard, |
| 1150 |
&mut submit_surface_textures_owned, |
| 1151 |
&mut used_surface_textures, |
| 1152 |
&mut command_index_guard, |
| 1153 |
); |
| 1154 |
if let Err(err) = res { |
| 1155 |
first_error.get_or_insert(err); |
| 1156 |
continue; |
| 1157 |
} |
| 1158 |
cmd_buf_data.into_baked_commands() |
| 1159 |
} |
| 1160 |
Err(err) => { |
| 1161 |
first_error.get_or_insert(err.into()); |
| 1162 |
continue; |
| 1163 |
} |
| 1164 |
}; |
| 1165 |
|
| 1166 |
// execute resource transitions |
| 1167 |
if let Err(e) = baked.encoder.open_pass(Some("(wgpu internal) Transit")) { |
| 1168 |
break 'error Err(e.into()); |
| 1169 |
} |
| 1170 |
|
| 1171 |
//Note: locking the trackers has to be done after the storages |
| 1172 |
let mut trackers = self.device.trackers.lock(); |
| 1173 |
if let Err(e) = baked.initialize_buffer_memory(&mut trackers, &snatch_guard) |
| 1174 |
{ |
| 1175 |
break 'error Err(e.into()); |
| 1176 |
} |
| 1177 |
if let Err(e) = baked.initialize_texture_memory( |
| 1178 |
&mut trackers, |
| 1179 |
&self.device, |
| 1180 |
&snatch_guard, |
| 1181 |
) { |
| 1182 |
break 'error Err(e.into()); |
| 1183 |
} |
| 1184 |
|
| 1185 |
//Note: stateless trackers are not merged: |
| 1186 |
// device already knows these resources exist. |
| 1187 |
CommandBuffer::insert_barriers_from_device_tracker( |
| 1188 |
baked.encoder.raw.as_mut(), |
| 1189 |
&mut trackers, |
| 1190 |
&baked.trackers, |
| 1191 |
&snatch_guard, |
| 1192 |
); |
| 1193 |
|
| 1194 |
if let Err(e) = baked.encoder.close_and_push_front() { |
| 1195 |
break 'error Err(e.into()); |
| 1196 |
} |
| 1197 |
|
| 1198 |
// Transition surface textures into `Present` state. |
| 1199 |
// Note: we could technically do it after all of the command buffers, |
| 1200 |
// but here we have a command encoder by hand, so it's easier to use it. |
| 1201 |
if !used_surface_textures.is_empty() { |
| 1202 |
if let Err(e) = baked.encoder.open_pass(Some("(wgpu internal) Present")) |
| 1203 |
{ |
| 1204 |
break 'error Err(e.into()); |
| 1205 |
} |
| 1206 |
let texture_barriers = trackers |
| 1207 |
.textures |
| 1208 |
.set_from_usage_scope_and_drain_transitions( |
| 1209 |
&used_surface_textures, |
| 1210 |
&snatch_guard, |
| 1211 |
) |
| 1212 |
.collect::<Vec<_>>(); |
| 1213 |
unsafe { |
| 1214 |
baked.encoder.raw.transition_textures(&texture_barriers); |
| 1215 |
}; |
| 1216 |
if let Err(e) = baked.encoder.close() { |
| 1217 |
break 'error Err(e.into()); |
| 1218 |
} |
| 1219 |
used_surface_textures = track::TextureUsageScope::default(); |
| 1220 |
} |
| 1221 |
|
| 1222 |
// done |
| 1223 |
active_executions.push(EncoderInFlight { |
| 1224 |
inner: baked.encoder, |
| 1225 |
trackers: baked.trackers, |
| 1226 |
temp_resources: baked.temp_resources, |
| 1227 |
_indirect_draw_validation_resources: baked |
| 1228 |
.indirect_draw_validation_resources, |
| 1229 |
pending_buffers: FastHashMap::default(), |
| 1230 |
pending_textures: FastHashMap::default(), |
| 1231 |
}); |
| 1232 |
} |
| 1233 |
|
| 1234 |
if let Some(first_error) = first_error { |
| 1235 |
break 'error Err(first_error); |
| 1236 |
} |
| 1237 |
} |
| 1238 |
} |
| 1239 |
|
| 1240 |
let mut pending_writes = self.pending_writes.lock(); |
| 1241 |
|
| 1242 |
{ |
| 1243 |
used_surface_textures.set_size(self.device.tracker_indices.textures.size()); |
| 1244 |
for texture in pending_writes.dst_textures.values() { |
| 1245 |
match texture.try_inner(&snatch_guard) { |
| 1246 |
Ok(TextureInner::Native { .. }) => {} |
| 1247 |
Ok(TextureInner::Surface { .. }) => { |
| 1248 |
// Compare the Arcs by pointer as Textures don't implement Eq |
| 1249 |
submit_surface_textures_owned |
| 1250 |
.insert(Arc::as_ptr(texture), texture.clone()); |
| 1251 |
|
| 1252 |
unsafe { |
| 1253 |
used_surface_textures |
| 1254 |
.merge_single(texture, None, wgt::TextureUses::PRESENT) |
| 1255 |
.unwrap() |
| 1256 |
}; |
| 1257 |
} |
| 1258 |
Err(e) => break 'error Err(e.into()), |
| 1259 |
} |
| 1260 |
} |
| 1261 |
|
| 1262 |
if !used_surface_textures.is_empty() { |
| 1263 |
let mut trackers = self.device.trackers.lock(); |
| 1264 |
|
| 1265 |
let texture_barriers = trackers |
| 1266 |
.textures |
| 1267 |
.set_from_usage_scope_and_drain_transitions( |
| 1268 |
&used_surface_textures, |
| 1269 |
&snatch_guard, |
| 1270 |
) |
| 1271 |
.collect::<Vec<_>>(); |
| 1272 |
unsafe { |
| 1273 |
pending_writes |
| 1274 |
.command_encoder |
| 1275 |
.transition_textures(&texture_barriers); |
| 1276 |
}; |
| 1277 |
} |
| 1278 |
} |
| 1279 |
|
| 1280 |
match pending_writes.pre_submit(&self.device.command_allocator, &self.device, self) { |
| 1281 |
Ok(Some(pending_execution)) => { |
| 1282 |
active_executions.insert(0, pending_execution); |
| 1283 |
} |
| 1284 |
Ok(None) => {} |
| 1285 |
Err(e) => break 'error Err(e.into()), |
| 1286 |
} |
| 1287 |
let hal_command_buffers = active_executions |
| 1288 |
.iter() |
| 1289 |
.flat_map(|e| e.inner.list.iter().map(|b| b.as_ref())) |
| 1290 |
.collect::<Vec<_>>(); |
| 1291 |
|
| 1292 |
{ |
| 1293 |
let mut submit_surface_textures = |
| 1294 |
SmallVec::<[&dyn hal::DynSurfaceTexture; 2]>::with_capacity( |
| 1295 |
submit_surface_textures_owned.len(), |
| 1296 |
); |
| 1297 |
|
| 1298 |
for texture in submit_surface_textures_owned.values() { |
| 1299 |
let raw = match texture.inner.get(&snatch_guard) { |
| 1300 |
Some(TextureInner::Surface { raw, .. }) => raw.as_ref(), |
| 1301 |
_ => unreachable!(), |
| 1302 |
}; |
| 1303 |
submit_surface_textures.push(raw); |
| 1304 |
} |
| 1305 |
|
| 1306 |
if let Err(e) = unsafe { |
| 1307 |
self.raw().submit( |
| 1308 |
&hal_command_buffers, |
| 1309 |
&submit_surface_textures, |
| 1310 |
(fence.as_mut(), submit_index), |
| 1311 |
) |
| 1312 |
} |
| 1313 |
.map_err(|e| self.device.handle_hal_error(e)) |
| 1314 |
{ |
| 1315 |
break 'error Err(e.into()); |
| 1316 |
} |
| 1317 |
|
| 1318 |
drop(command_index_guard); |
| 1319 |
|
| 1320 |
// Advance the successful submission index. |
| 1321 |
self.device |
| 1322 |
.last_successful_submission_index |
| 1323 |
.fetch_max(submit_index, Ordering::SeqCst); |
| 1324 |
} |
| 1325 |
|
| 1326 |
profiling::scope!("cleanup"); |
| 1327 |
|
| 1328 |
// this will register the new submission to the life time tracker |
| 1329 |
self.lock_life() |
| 1330 |
.track_submission(submit_index, active_executions); |
| 1331 |
drop(pending_writes); |
| 1332 |
|
| 1333 |
// This will schedule destruction of all resources that are no longer needed |
| 1334 |
// by the user but used in the command stream, among other things. |
| 1335 |
let fence_guard = RwLockWriteGuard::downgrade(fence); |
| 1336 |
let (closures, result) = |
| 1337 |
self.device |
| 1338 |
.maintain(fence_guard, wgt::PollType::Poll, snatch_guard); |
| 1339 |
match result { |
| 1340 |
Ok(status) => { |
| 1341 |
debug_assert!(matches!( |
| 1342 |
status, |
| 1343 |
wgt::PollStatus::QueueEmpty | wgt::PollStatus::Poll |
| 1344 |
)); |
| 1345 |
} |
| 1346 |
Err(WaitIdleError::Device(err)) => break 'error Err(QueueSubmitError::Queue(err)), |
| 1347 |
Err(WaitIdleError::WrongSubmissionIndex(..)) => { |
| 1348 |
unreachable!("Cannot get WrongSubmissionIndex from Poll") |
| 1349 |
} |
| 1350 |
Err(WaitIdleError::Timeout) => unreachable!("Cannot get Timeout from Poll"), |
| 1351 |
}; |
| 1352 |
|
| 1353 |
Ok(closures) |
| 1354 |
}; |
| 1355 |
|
| 1356 |
let callbacks = match res { |
| 1357 |
Ok(ok) => ok, |
| 1358 |
Err(e) => return Err((submit_index, e)), |
| 1359 |
}; |
| 1360 |
|
| 1361 |
// the closures should execute with nothing locked! |
| 1362 |
callbacks.fire(); |
| 1363 |
|
| 1364 |
self.device.lose_if_oom(); |
| 1365 |
|
| 1366 |
api_log!("Queue::submit returned submit index {submit_index}"); |
| 1367 |
|
| 1368 |
Ok(submit_index) |
| 1369 |
} |
| 1370 |
|
| 1371 |
pub fn get_timestamp_period(&self) -> f32 { |
| 1372 |
unsafe { self.raw().get_timestamp_period() } |
| 1373 |
} |
| 1374 |
|
| 1375 |
/// `closure` is guaranteed to be called. |
| 1376 |
pub fn on_submitted_work_done( |
| 1377 |
&self, |
| 1378 |
closure: SubmittedWorkDoneClosure, |
| 1379 |
) -> Option<SubmissionIndex> { |
| 1380 |
api_log!("Queue::on_submitted_work_done"); |
| 1381 |
//TODO: flush pending writes |
| 1382 |
self.lock_life().add_work_done_closure(closure) |
| 1383 |
} |
| 1384 |
} |
| 1385 |
|
| 1386 |
impl Global { |
| 1387 |
pub fn queue_write_buffer( |
| 1388 |
&self, |
| 1389 |
queue_id: QueueId, |
| 1390 |
buffer_id: id::BufferId, |
| 1391 |
buffer_offset: wgt::BufferAddress, |
| 1392 |
data: &[u8], |
| 1393 |
) -> Result<(), QueueWriteError> { |
| 1394 |
let queue = self.hub.queues.get(queue_id); |
| 1395 |
|
| 1396 |
#[cfg(feature = "trace")] |
| 1397 |
if let Some(ref mut trace) = *queue.device.trace.lock() { |
| 1398 |
let data_path = trace.make_binary("bin", data); |
| 1399 |
trace.add(Action::WriteBuffer { |
| 1400 |
id: buffer_id, |
| 1401 |
data: data_path, |
| 1402 |
range: buffer_offset..buffer_offset + data.len() as u64, |
| 1403 |
queued: true, |
| 1404 |
}); |
| 1405 |
} |
| 1406 |
|
| 1407 |
let buffer = self.hub.buffers.get(buffer_id); |
| 1408 |
queue.write_buffer(buffer, buffer_offset, data) |
| 1409 |
} |
| 1410 |
|
| 1411 |
pub fn queue_create_staging_buffer( |
| 1412 |
&self, |
| 1413 |
queue_id: QueueId, |
| 1414 |
buffer_size: wgt::BufferSize, |
| 1415 |
id_in: Option<id::StagingBufferId>, |
| 1416 |
) -> Result<(id::StagingBufferId, NonNull<u8>), QueueWriteError> { |
| 1417 |
let queue = self.hub.queues.get(queue_id); |
| 1418 |
let (staging_buffer, ptr) = queue.create_staging_buffer(buffer_size)?; |
| 1419 |
|
| 1420 |
let fid = self.hub.staging_buffers.prepare(id_in); |
| 1421 |
let id = fid.assign(staging_buffer); |
| 1422 |
|
| 1423 |
Ok((id, ptr)) |
| 1424 |
} |
| 1425 |
|
| 1426 |
pub fn queue_write_staging_buffer( |
| 1427 |
&self, |
| 1428 |
queue_id: QueueId, |
| 1429 |
buffer_id: id::BufferId, |
| 1430 |
buffer_offset: wgt::BufferAddress, |
| 1431 |
staging_buffer_id: id::StagingBufferId, |
| 1432 |
) -> Result<(), QueueWriteError> { |
| 1433 |
let queue = self.hub.queues.get(queue_id); |
| 1434 |
let buffer = self.hub.buffers.get(buffer_id); |
| 1435 |
let staging_buffer = self.hub.staging_buffers.remove(staging_buffer_id); |
| 1436 |
queue.write_staging_buffer(buffer, buffer_offset, staging_buffer) |
| 1437 |
} |
| 1438 |
|
| 1439 |
pub fn queue_validate_write_buffer( |
| 1440 |
&self, |
| 1441 |
queue_id: QueueId, |
| 1442 |
buffer_id: id::BufferId, |
| 1443 |
buffer_offset: u64, |
| 1444 |
buffer_size: wgt::BufferSize, |
| 1445 |
) -> Result<(), QueueWriteError> { |
| 1446 |
let queue = self.hub.queues.get(queue_id); |
| 1447 |
let buffer = self.hub.buffers.get(buffer_id); |
| 1448 |
queue.validate_write_buffer(buffer, buffer_offset, buffer_size) |
| 1449 |
} |
| 1450 |
|
| 1451 |
pub fn queue_write_texture( |
| 1452 |
&self, |
| 1453 |
queue_id: QueueId, |
| 1454 |
destination: &TexelCopyTextureInfo, |
| 1455 |
data: &[u8], |
| 1456 |
data_layout: &wgt::TexelCopyBufferLayout, |
| 1457 |
size: &wgt::Extent3d, |
| 1458 |
) -> Result<(), QueueWriteError> { |
| 1459 |
let queue = self.hub.queues.get(queue_id); |
| 1460 |
|
| 1461 |
#[cfg(feature = "trace")] |
| 1462 |
if let Some(ref mut trace) = *queue.device.trace.lock() { |
| 1463 |
let data_path = trace.make_binary("bin", data); |
| 1464 |
trace.add(Action::WriteTexture { |
| 1465 |
to: *destination, |
| 1466 |
data: data_path, |
| 1467 |
layout: *data_layout, |
| 1468 |
size: *size, |
| 1469 |
}); |
| 1470 |
} |
| 1471 |
|
| 1472 |
let destination = wgt::TexelCopyTextureInfo { |
| 1473 |
texture: self.hub.textures.get(destination.texture), |
| 1474 |
mip_level: destination.mip_level, |
| 1475 |
origin: destination.origin, |
| 1476 |
aspect: destination.aspect, |
| 1477 |
}; |
| 1478 |
queue.write_texture(destination, data, data_layout, size) |
| 1479 |
} |
| 1480 |
|
| 1481 |
#[cfg(webgl)] |
| 1482 |
pub fn queue_copy_external_image_to_texture( |
| 1483 |
&self, |
| 1484 |
queue_id: QueueId, |
| 1485 |
source: &wgt::CopyExternalImageSourceInfo, |
| 1486 |
destination: crate::command::CopyExternalImageDestInfo, |
| 1487 |
size: wgt::Extent3d, |
| 1488 |
) -> Result<(), QueueWriteError> { |
| 1489 |
let queue = self.hub.queues.get(queue_id); |
| 1490 |
let destination = wgt::CopyExternalImageDestInfo { |
| 1491 |
texture: self.hub.textures.get(destination.texture), |
| 1492 |
mip_level: destination.mip_level, |
| 1493 |
origin: destination.origin, |
| 1494 |
aspect: destination.aspect, |
| 1495 |
color_space: destination.color_space, |
| 1496 |
premultiplied_alpha: destination.premultiplied_alpha, |
| 1497 |
}; |
| 1498 |
queue.copy_external_image_to_texture(source, destination, size) |
| 1499 |
} |
| 1500 |
|
| 1501 |
pub fn queue_submit( |
| 1502 |
&self, |
| 1503 |
queue_id: QueueId, |
| 1504 |
command_buffer_ids: &[id::CommandBufferId], |
| 1505 |
) -> Result<SubmissionIndex, (SubmissionIndex, QueueSubmitError)> { |
| 1506 |
let queue = self.hub.queues.get(queue_id); |
| 1507 |
let command_buffer_guard = self.hub.command_buffers.read(); |
| 1508 |
let command_buffers = command_buffer_ids |
| 1509 |
.iter() |
| 1510 |
.map(|id| command_buffer_guard.get(*id)) |
| 1511 |
.collect::<Vec<_>>(); |
| 1512 |
drop(command_buffer_guard); |
| 1513 |
queue.submit(&command_buffers) |
| 1514 |
} |
| 1515 |
|
| 1516 |
pub fn queue_get_timestamp_period(&self, queue_id: QueueId) -> f32 { |
| 1517 |
let queue = self.hub.queues.get(queue_id); |
| 1518 |
|
| 1519 |
if queue.device.timestamp_normalizer.get().unwrap().enabled() { |
| 1520 |
return 1.0; |
| 1521 |
} |
| 1522 |
|
| 1523 |
queue.get_timestamp_period() |
| 1524 |
} |
| 1525 |
|
| 1526 |
pub fn queue_on_submitted_work_done( |
| 1527 |
&self, |
| 1528 |
queue_id: QueueId, |
| 1529 |
closure: SubmittedWorkDoneClosure, |
| 1530 |
) -> SubmissionIndex { |
| 1531 |
api_log!("Queue::on_submitted_work_done {queue_id:?}"); |
| 1532 |
|
| 1533 |
//TODO: flush pending writes |
| 1534 |
let queue = self.hub.queues.get(queue_id); |
| 1535 |
let result = queue.on_submitted_work_done(closure); |
| 1536 |
result.unwrap_or(0) // '0' means no wait is necessary |
| 1537 |
} |
| 1538 |
} |
| 1539 |
|
| 1540 |
fn validate_command_buffer( |
| 1541 |
command_buffer: &CommandBuffer, |
| 1542 |
queue: &Queue, |
| 1543 |
cmd_buf_data: &crate::command::CommandBufferMutable, |
| 1544 |
snatch_guard: &SnatchGuard, |
| 1545 |
submit_surface_textures_owned: &mut FastHashMap<*const Texture, Arc<Texture>>, |
| 1546 |
used_surface_textures: &mut track::TextureUsageScope, |
| 1547 |
command_index_guard: &mut RwLockWriteGuard<CommandIndices>, |
| 1548 |
) -> Result<(), QueueSubmitError> { |
| 1549 |
command_buffer.same_device_as(queue)?; |
| 1550 |
|
| 1551 |
{ |
| 1552 |
profiling::scope!("check resource state"); |
| 1553 |
|
| 1554 |
{ |
| 1555 |
profiling::scope!("buffers"); |
| 1556 |
for buffer in cmd_buf_data.trackers.buffers.used_resources() { |
| 1557 |
buffer.check_destroyed(snatch_guard)?; |
| 1558 |
|
| 1559 |
match *buffer.map_state.lock() { |
| 1560 |
BufferMapState::Idle => (), |
| 1561 |
_ => return Err(QueueSubmitError::BufferStillMapped(buffer.error_ident())), |
| 1562 |
} |
| 1563 |
} |
| 1564 |
} |
| 1565 |
{ |
| 1566 |
profiling::scope!("textures"); |
| 1567 |
for texture in cmd_buf_data.trackers.textures.used_resources() { |
| 1568 |
let should_extend = match texture.try_inner(snatch_guard)? { |
| 1569 |
TextureInner::Native { .. } => false, |
| 1570 |
TextureInner::Surface { .. } => { |
| 1571 |
// Compare the Arcs by pointer as Textures don't implement Eq. |
| 1572 |
submit_surface_textures_owned.insert(Arc::as_ptr(texture), texture.clone()); |
| 1573 |
|
| 1574 |
true |
| 1575 |
} |
| 1576 |
}; |
| 1577 |
if should_extend { |
| 1578 |
unsafe { |
| 1579 |
used_surface_textures |
| 1580 |
.merge_single(texture, None, wgt::TextureUses::PRESENT) |
| 1581 |
.unwrap(); |
| 1582 |
}; |
| 1583 |
} |
| 1584 |
} |
| 1585 |
} |
| 1586 |
|
| 1587 |
if let Err(e) = |
| 1588 |
cmd_buf_data.validate_acceleration_structure_actions(snatch_guard, command_index_guard) |
| 1589 |
{ |
| 1590 |
return Err(e.into()); |
| 1591 |
} |
| 1592 |
} |
| 1593 |
Ok(()) |
| 1594 |
} |