Debian Codegraph @

third_party/rust/wgpu-core/src/device/queue.rs

raw
1 use alloc::{boxed::Box, sync::Arc, vec, vec::Vec};
2 use core::{
3 iter,
4 mem::{self, ManuallyDrop},
5 ptr::NonNull,
6 sync::atomic::Ordering,
7 };
8
9 use smallvec::SmallVec;
10 use thiserror::Error;
11
12 use super::{life::LifetimeTracker, Device};
13 use crate::device::resource::CommandIndices;
14 #[cfg(feature = "trace")]
15 use crate::device::trace::Action;
16 use crate::scratch::ScratchBuffer;
17 use crate::{
18 api_log,
19 command::{
20 extract_texture_selector, validate_linear_texture_data, validate_texture_copy_range,
21 ClearError, CommandAllocator, CommandBuffer, CommandEncoderError, CopySide,
22 TexelCopyTextureInfo, TransferError,
23 },
24 conv,
25 device::{DeviceError, WaitIdleError},
26 get_lowest_common_denom,
27 global::Global,
28 id::{self, QueueId},
29 init_tracker::{has_copy_partial_init_tracker_coverage, TextureInitRange},
30 lock::{rank, Mutex, MutexGuard, RwLockWriteGuard},
31 resource::{
32 Buffer, BufferAccessError, BufferMapState, DestroyedBuffer, DestroyedResourceError,
33 DestroyedTexture, Fallible, FlushedStagingBuffer, InvalidResourceError, Labeled,
34 ParentDevice, ResourceErrorIdent, StagingBuffer, Texture, TextureInner, Trackable,
35 },
36 resource_log,
37 snatch::SnatchGuard,
38 track::{self, Tracker, TrackerIndex},
39 FastHashMap, SubmissionIndex,
40 };
41
42 pub struct Queue {
43 raw: Box<dyn hal::DynQueue>,
44 pub(crate) pending_writes: Mutex<PendingWrites>,
45 life_tracker: Mutex<LifetimeTracker>,
46 // The device needs to be dropped last (`Device.zero_buffer` might be referenced by the encoder in pending writes).
47 pub(crate) device: Arc<Device>,
48 }
49
50 impl Queue {
51 pub(crate) fn new(
52 device: Arc<Device>,
53 raw: Box<dyn hal::DynQueue>,
54 ) -> Result<Self, DeviceError> {
55 let pending_encoder = device
56 .command_allocator
57 .acquire_encoder(device.raw(), raw.as_ref())
58 .map_err(DeviceError::from_hal);
59
60 let pending_encoder = match pending_encoder {
61 Ok(pending_encoder) => pending_encoder,
62 Err(e) => {
63 return Err(e);
64 }
65 };
66
67 let mut pending_writes = PendingWrites::new(pending_encoder);
68
69 let zero_buffer = device.zero_buffer.as_ref();
70 pending_writes.activate();
71 unsafe {
72 pending_writes
73 .command_encoder
74 .transition_buffers(&[hal::BufferBarrier {
75 buffer: zero_buffer,
76 usage: hal::StateTransition {
77 from: wgt::BufferUses::empty(),
78 to: wgt::BufferUses::COPY_DST,
79 },
80 }]);
81 pending_writes
82 .command_encoder
83 .clear_buffer(zero_buffer, 0..super::ZERO_BUFFER_SIZE);
84 pending_writes
85 .command_encoder
86 .transition_buffers(&[hal::BufferBarrier {
87 buffer: zero_buffer,
88 usage: hal::StateTransition {
89 from: wgt::BufferUses::COPY_DST,
90 to: wgt::BufferUses::COPY_SRC,
91 },
92 }]);
93 }
94
95 Ok(Queue {
96 raw,
97 device,
98 pending_writes: Mutex::new(rank::QUEUE_PENDING_WRITES, pending_writes),
99 life_tracker: Mutex::new(rank::QUEUE_LIFE_TRACKER, LifetimeTracker::new()),
100 })
101 }
102
103 pub(crate) fn raw(&self) -> &dyn hal::DynQueue {
104 self.raw.as_ref()
105 }
106
107 #[track_caller]
108 pub(crate) fn lock_life<'a>(&'a self) -> MutexGuard<'a, LifetimeTracker> {
109 self.life_tracker.lock()
110 }
111
112 pub(crate) fn maintain(
113 &self,
114 submission_index: u64,
115 snatch_guard: &SnatchGuard,
116 ) -> (
117 SmallVec<[SubmittedWorkDoneClosure; 1]>,
118 Vec<super::BufferMapPendingClosure>,
119 bool,
120 ) {
121 let mut life_tracker = self.lock_life();
122 let submission_closures = life_tracker.triage_submissions(submission_index);
123
124 let mapping_closures = life_tracker.handle_mapping(snatch_guard);
125
126 let queue_empty = life_tracker.queue_empty();
127
128 (submission_closures, mapping_closures, queue_empty)
129 }
130 }
131
132 crate::impl_resource_type!(Queue);
133 // TODO: https://github.com/gfx-rs/wgpu/issues/4014
134 impl Labeled for Queue {
135 fn label(&self) -> &str {
136 ""
137 }
138 }
139 crate::impl_parent_device!(Queue);
140 crate::impl_storage_item!(Queue);
141
142 impl Drop for Queue {
143 fn drop(&mut self) {
144 resource_log!("Drop {}", self.error_ident());
145
146 let last_successful_submission_index = self
147 .device
148 .last_successful_submission_index
149 .load(Ordering::Acquire);
150
151 let fence = self.device.fence.read();
152
153 // Try waiting on the last submission using the following sequence of timeouts
154 let timeouts_in_ms = [100, 200, 400, 800, 1600, 3200];
155
156 for (i, timeout_ms) in timeouts_in_ms.into_iter().enumerate() {
157 let is_last_iter = i == timeouts_in_ms.len() - 1;
158
159 api_log!(
160 "Waiting on last submission. try: {}/{}. timeout: {}ms",
161 i + 1,
162 timeouts_in_ms.len(),
163 timeout_ms
164 );
165
166 let wait_res = unsafe {
167 self.device.raw().wait(
168 fence.as_ref(),
169 last_successful_submission_index,
170 #[cfg(not(target_arch = "wasm32"))]
171 timeout_ms,
172 #[cfg(target_arch = "wasm32")]
173 0, // WebKit and Chromium don't support a non-0 timeout
174 )
175 };
176 // Note: If we don't panic below we are in UB land (destroying resources while they are still in use by the GPU).
177 match wait_res {
178 Ok(true) => break,
179 Ok(false) => {
180 // It's fine that we timed out on WebGL; GL objects can be deleted early as they
181 // will be kept around by the driver if GPU work hasn't finished.
182 // Moreover, the way we emulate read mappings on WebGL allows us to execute map_buffer earlier than on other
183 // backends since getBufferSubData is synchronous with respect to the other previously enqueued GL commands.
184 // Relying on this behavior breaks the clean abstraction wgpu-hal tries to maintain and
185 // we should find ways to improve this. See https://github.com/gfx-rs/wgpu/issues/6538.
186 #[cfg(target_arch = "wasm32")]
187 {
188 break;
189 }
190 #[cfg(not(target_arch = "wasm32"))]
191 {
192 if is_last_iter {
193 panic!(
194 "We timed out while waiting on the last successful submission to complete!"
195 );
196 }
197 }
198 }
199 Err(e) => match e {
200 hal::DeviceError::OutOfMemory => {
201 if is_last_iter {
202 panic!(
203 "We ran into an OOM error while waiting on the last successful submission to complete!"
204 );
205 }
206 }
207 hal::DeviceError::Lost => {
208 self.device.handle_hal_error(e); // will lose the device
209 break;
210 }
211 hal::DeviceError::ResourceCreationFailed => unreachable!(),
212 hal::DeviceError::Unexpected => {
213 panic!(
214 "We ran into an unexpected error while waiting on the last successful submission to complete!"
215 );
216 }
217 },
218 }
219 }
220 drop(fence);
221
222 let snatch_guard = self.device.snatchable_lock.read();
223 let (submission_closures, mapping_closures, queue_empty) =
224 self.maintain(last_successful_submission_index, &snatch_guard);
225 drop(snatch_guard);
226
227 assert!(queue_empty);
228
229 let closures = crate::device::UserClosures {
230 mappings: mapping_closures,
231 submissions: submission_closures,
232 device_lost_invocations: SmallVec::new(),
233 };
234
235 closures.fire();
236 }
237 }
238
239 #[cfg(send_sync)]
240 pub type SubmittedWorkDoneClosure = Box<dyn FnOnce() + Send + 'static>;
241 #[cfg(not(send_sync))]
242 pub type SubmittedWorkDoneClosure = Box<dyn FnOnce() + 'static>;
243
244 /// A texture or buffer to be freed soon.
245 ///
246 /// This is just a tagged raw texture or buffer, generally about to be added to
247 /// some other more specific container like:
248 ///
249 /// - `PendingWrites::temp_resources`: resources used by queue writes and
250 /// unmaps, waiting to be folded in with the next queue submission
251 ///
252 /// - `ActiveSubmission::temp_resources`: temporary resources used by a queue
253 /// submission, to be freed when it completes
254 #[derive(Debug)]
255 pub enum TempResource {
256 StagingBuffer(FlushedStagingBuffer),
257 ScratchBuffer(ScratchBuffer),
258 DestroyedBuffer(DestroyedBuffer),
259 DestroyedTexture(DestroyedTexture),
260 }
261
262 /// A series of raw [`CommandBuffer`]s that have been submitted to a
263 /// queue, and the [`wgpu_hal::CommandEncoder`] that built them.
264 ///
265 /// [`CommandBuffer`]: hal::Api::CommandBuffer
266 /// [`wgpu_hal::CommandEncoder`]: hal::CommandEncoder
267 pub(crate) struct EncoderInFlight {
268 inner: crate::command::CommandEncoder,
269 pub(crate) trackers: Tracker,
270 pub(crate) temp_resources: Vec<TempResource>,
271 /// We only need to keep these resources alive.
272 _indirect_draw_validation_resources: crate::indirect_validation::DrawResources,
273
274 /// These are the buffers that have been tracked by `PendingWrites`.
275 pub(crate) pending_buffers: FastHashMap<TrackerIndex, Arc<Buffer>>,
276 /// These are the textures that have been tracked by `PendingWrites`.
277 pub(crate) pending_textures: FastHashMap<TrackerIndex, Arc<Texture>>,
278 }
279
280 /// A private command encoder for writes made directly on the device
281 /// or queue.
282 ///
283 /// Operations like `buffer_unmap`, `queue_write_buffer`, and
284 /// `queue_write_texture` need to copy data to the GPU. At the hal
285 /// level, this must be done by encoding and submitting commands, but
286 /// these operations are not associated with any specific wgpu command
287 /// buffer.
288 ///
289 /// Instead, `Device::pending_writes` owns one of these values, which
290 /// has its own hal command encoder and resource lists. The commands
291 /// accumulated here are automatically submitted to the queue the next
292 /// time the user submits a wgpu command buffer, ahead of the user's
293 /// commands.
294 ///
295 /// Important:
296 /// When locking pending_writes be sure that tracker is not locked
297 /// and try to lock trackers for the minimum timespan possible
298 ///
299 /// All uses of [`StagingBuffer`]s end up here.
300 #[derive(Debug)]
301 pub(crate) struct PendingWrites {
302 // The command encoder needs to be destroyed before any other resource in pending writes.
303 pub command_encoder: Box<dyn hal::DynCommandEncoder>,
304
305 /// True if `command_encoder` is in the "recording" state, as
306 /// described in the docs for the [`wgpu_hal::CommandEncoder`]
307 /// trait.
308 ///
309 /// [`wgpu_hal::CommandEncoder`]: hal::CommandEncoder
310 pub is_recording: bool,
311
312 temp_resources: Vec<TempResource>,
313 dst_buffers: FastHashMap<TrackerIndex, Arc<Buffer>>,
314 dst_textures: FastHashMap<TrackerIndex, Arc<Texture>>,
315 }
316
317 impl PendingWrites {
318 pub fn new(command_encoder: Box<dyn hal::DynCommandEncoder>) -> Self {
319 Self {
320 command_encoder,
321 is_recording: false,
322 temp_resources: Vec::new(),
323 dst_buffers: FastHashMap::default(),
324 dst_textures: FastHashMap::default(),
325 }
326 }
327
328 pub fn insert_buffer(&mut self, buffer: &Arc<Buffer>) {
329 self.dst_buffers
330 .insert(buffer.tracker_index(), buffer.clone());
331 }
332
333 pub fn insert_texture(&mut self, texture: &Arc<Texture>) {
334 self.dst_textures
335 .insert(texture.tracker_index(), texture.clone());
336 }
337
338 pub fn contains_buffer(&self, buffer: &Arc<Buffer>) -> bool {
339 self.dst_buffers.contains_key(&buffer.tracker_index())
340 }
341
342 pub fn contains_texture(&self, texture: &Arc<Texture>) -> bool {
343 self.dst_textures.contains_key(&texture.tracker_index())
344 }
345
346 pub fn consume_temp(&mut self, resource: TempResource) {
347 self.temp_resources.push(resource);
348 }
349
350 pub fn consume(&mut self, buffer: FlushedStagingBuffer) {
351 self.temp_resources
352 .push(TempResource::StagingBuffer(buffer));
353 }
354
355 fn pre_submit(
356 &mut self,
357 command_allocator: &CommandAllocator,
358 device: &Arc<Device>,
359 queue: &Queue,
360 ) -> Result<Option<EncoderInFlight>, DeviceError> {
361 if self.is_recording {
362 let pending_buffers = mem::take(&mut self.dst_buffers);
363 let pending_textures = mem::take(&mut self.dst_textures);
364
365 let cmd_buf = unsafe { self.command_encoder.end_encoding() }
366 .map_err(|e| device.handle_hal_error(e))?;
367 self.is_recording = false;
368
369 let new_encoder = command_allocator
370 .acquire_encoder(device.raw(), queue.raw())
371 .map_err(|e| device.handle_hal_error(e))?;
372
373 let encoder = EncoderInFlight {
374 inner: crate::command::CommandEncoder {
375 raw: ManuallyDrop::new(mem::replace(&mut self.command_encoder, new_encoder)),
376 list: vec![cmd_buf],
377 device: device.clone(),
378 is_open: false,
379 hal_label: None,
380 },
381 trackers: Tracker::new(),
382 temp_resources: mem::take(&mut self.temp_resources),
383 _indirect_draw_validation_resources: crate::indirect_validation::DrawResources::new(
384 device.clone(),
385 ),
386 pending_buffers,
387 pending_textures,
388 };
389 Ok(Some(encoder))
390 } else {
391 self.dst_buffers.clear();
392 self.dst_textures.clear();
393 Ok(None)
394 }
395 }
396
397 pub fn activate(&mut self) -> &mut dyn hal::DynCommandEncoder {
398 if !self.is_recording {
399 unsafe {
400 self.command_encoder
401 .begin_encoding(Some("(wgpu internal) PendingWrites"))
402 .unwrap();
403 }
404 self.is_recording = true;
405 }
406 self.command_encoder.as_mut()
407 }
408 }
409
410 impl Drop for PendingWrites {
411 fn drop(&mut self) {
412 unsafe {
413 if self.is_recording {
414 self.command_encoder.discard_encoding();
415 }
416 }
417 }
418 }
419
420 #[derive(Clone, Debug, Error)]
421 #[non_exhaustive]
422 pub enum QueueWriteError {
423 #[error(transparent)]
424 Queue(#[from] DeviceError),
425 #[error(transparent)]
426 Transfer(#[from] TransferError),
427 #[error(transparent)]
428 MemoryInitFailure(#[from] ClearError),
429 #[error(transparent)]
430 DestroyedResource(#[from] DestroyedResourceError),
431 #[error(transparent)]
432 InvalidResource(#[from] InvalidResourceError),
433 }
434
435 #[derive(Clone, Debug, Error)]
436 #[non_exhaustive]
437 pub enum QueueSubmitError {
438 #[error(transparent)]
439 Queue(#[from] DeviceError),
440 #[error(transparent)]
441 DestroyedResource(#[from] DestroyedResourceError),
442 #[error(transparent)]
443 Unmap(#[from] BufferAccessError),
444 #[error("{0} is still mapped")]
445 BufferStillMapped(ResourceErrorIdent),
446 #[error(transparent)]
447 InvalidResource(#[from] InvalidResourceError),
448 #[error(transparent)]
449 CommandEncoder(#[from] CommandEncoderError),
450 #[error(transparent)]
451 ValidateAsActionsError(#[from] crate::ray_tracing::ValidateAsActionsError),
452 }
453
454 //TODO: move out common parts of write_xxx.
455
456 impl Queue {
457 pub fn write_buffer(
458 &self,
459 buffer: Fallible<Buffer>,
460 buffer_offset: wgt::BufferAddress,
461 data: &[u8],
462 ) -> Result<(), QueueWriteError> {
463 profiling::scope!("Queue::write_buffer");
464 api_log!("Queue::write_buffer");
465
466 self.device.check_is_valid()?;
467
468 let buffer = buffer.get()?;
469
470 let data_size = data.len() as wgt::BufferAddress;
471
472 self.same_device_as(buffer.as_ref())?;
473
474 let data_size = if let Some(data_size) = wgt::BufferSize::new(data_size) {
475 data_size
476 } else {
477 log::trace!("Ignoring write_buffer of size 0");
478 return Ok(());
479 };
480
481 let snatch_guard = self.device.snatchable_lock.read();
482
483 // Platform validation requires that the staging buffer always be
484 // freed, even if an error occurs. All paths from here must call
485 // `device.pending_writes.consume`.
486 let mut staging_buffer = StagingBuffer::new(&self.device, data_size)?;
487 let mut pending_writes = self.pending_writes.lock();
488
489 let staging_buffer = {
490 profiling::scope!("copy");
491 staging_buffer.write(data);
492 staging_buffer.flush()
493 };
494
495 let result = self.write_staging_buffer_impl(
496 &snatch_guard,
497 &mut pending_writes,
498 &staging_buffer,
499 buffer,
500 buffer_offset,
501 );
502
503 pending_writes.consume(staging_buffer);
504 result
505 }
506
507 pub fn create_staging_buffer(
508 &self,
509 buffer_size: wgt::BufferSize,
510 ) -> Result<(StagingBuffer, NonNull<u8>), QueueWriteError> {
511 profiling::scope!("Queue::create_staging_buffer");
512 resource_log!("Queue::create_staging_buffer");
513
514 self.device.check_is_valid()?;
515
516 let staging_buffer = StagingBuffer::new(&self.device, buffer_size)?;
517 let ptr = unsafe { staging_buffer.ptr() };
518
519 Ok((staging_buffer, ptr))
520 }
521
522 pub fn write_staging_buffer(
523 &self,
524 buffer: Fallible<Buffer>,
525 buffer_offset: wgt::BufferAddress,
526 staging_buffer: StagingBuffer,
527 ) -> Result<(), QueueWriteError> {
528 profiling::scope!("Queue::write_staging_buffer");
529
530 self.device.check_is_valid()?;
531
532 let buffer = buffer.get()?;
533
534 let snatch_guard = self.device.snatchable_lock.read();
535 let mut pending_writes = self.pending_writes.lock();
536
537 // At this point, we have taken ownership of the staging_buffer from the
538 // user. Platform validation requires that the staging buffer always
539 // be freed, even if an error occurs. All paths from here must call
540 // `device.pending_writes.consume`.
541 let staging_buffer = staging_buffer.flush();
542
543 let result = self.write_staging_buffer_impl(
544 &snatch_guard,
545 &mut pending_writes,
546 &staging_buffer,
547 buffer,
548 buffer_offset,
549 );
550
551 pending_writes.consume(staging_buffer);
552 result
553 }
554
555 pub fn validate_write_buffer(
556 &self,
557 buffer: Fallible<Buffer>,
558 buffer_offset: u64,
559 buffer_size: wgt::BufferSize,
560 ) -> Result<(), QueueWriteError> {
561 profiling::scope!("Queue::validate_write_buffer");
562
563 self.device.check_is_valid()?;
564
565 let buffer = buffer.get()?;
566
567 self.validate_write_buffer_impl(&buffer, buffer_offset, buffer_size)?;
568
569 Ok(())
570 }
571
572 fn validate_write_buffer_impl(
573 &self,
574 buffer: &Buffer,
575 buffer_offset: u64,
576 buffer_size: wgt::BufferSize,
577 ) -> Result<(), TransferError> {
578 buffer.check_usage(wgt::BufferUsages::COPY_DST)?;
579 if buffer_size.get() % wgt::COPY_BUFFER_ALIGNMENT != 0 {
580 return Err(TransferError::UnalignedCopySize(buffer_size.get()));
581 }
582 if buffer_offset % wgt::COPY_BUFFER_ALIGNMENT != 0 {
583 return Err(TransferError::UnalignedBufferOffset(buffer_offset));
584 }
585 if buffer_offset + buffer_size.get() > buffer.size {
586 return Err(TransferError::BufferOverrun {
587 start_offset: buffer_offset,
588 end_offset: buffer_offset + buffer_size.get(),
589 buffer_size: buffer.size,
590 side: CopySide::Destination,
591 });
592 }
593
594 Ok(())
595 }
596
597 fn write_staging_buffer_impl(
598 &self,
599 snatch_guard: &SnatchGuard,
600 pending_writes: &mut PendingWrites,
601 staging_buffer: &FlushedStagingBuffer,
602 buffer: Arc<Buffer>,
603 buffer_offset: u64,
604 ) -> Result<(), QueueWriteError> {
605 self.device.check_is_valid()?;
606
607 let transition = {
608 let mut trackers = self.device.trackers.lock();
609 trackers
610 .buffers
611 .set_single(&buffer, wgt::BufferUses::COPY_DST)
612 };
613
614 let dst_raw = buffer.try_raw(snatch_guard)?;
615
616 self.same_device_as(buffer.as_ref())?;
617
618 self.validate_write_buffer_impl(&buffer, buffer_offset, staging_buffer.size)?;
619
620 let region = hal::BufferCopy {
621 src_offset: 0,
622 dst_offset: buffer_offset,
623 size: staging_buffer.size,
624 };
625 let barriers = iter::once(hal::BufferBarrier {
626 buffer: staging_buffer.raw(),
627 usage: hal::StateTransition {
628 from: wgt::BufferUses::MAP_WRITE,
629 to: wgt::BufferUses::COPY_SRC,
630 },
631 })
632 .chain(transition.map(|pending| pending.into_hal(&buffer, snatch_guard)))
633 .collect::<Vec<_>>();
634 let encoder = pending_writes.activate();
635 unsafe {
636 encoder.transition_buffers(&barriers);
637 encoder.copy_buffer_to_buffer(staging_buffer.raw(), dst_raw, &[region]);
638 }
639
640 pending_writes.insert_buffer(&buffer);
641
642 // Ensure the overwritten bytes are marked as initialized so
643 // they don't need to be nulled prior to mapping or binding.
644 {
645 buffer
646 .initialization_status
647 .write()
648 .drain(buffer_offset..(buffer_offset + staging_buffer.size.get()));
649 }
650
651 Ok(())
652 }
653
654 pub fn write_texture(
655 &self,
656 destination: wgt::TexelCopyTextureInfo<Fallible<Texture>>,
657 data: &[u8],
658 data_layout: &wgt::TexelCopyBufferLayout,
659 size: &wgt::Extent3d,
660 ) -> Result<(), QueueWriteError> {
661 profiling::scope!("Queue::write_texture");
662 api_log!("Queue::write_texture");
663
664 self.device.check_is_valid()?;
665
666 if size.width == 0 || size.height == 0 || size.depth_or_array_layers == 0 {
667 log::trace!("Ignoring write_texture of size 0");
668 return Ok(());
669 }
670
671 let dst = destination.texture.get()?;
672 let destination = wgt::TexelCopyTextureInfo {
673 texture: (),
674 mip_level: destination.mip_level,
675 origin: destination.origin,
676 aspect: destination.aspect,
677 };
678
679 self.same_device_as(dst.as_ref())?;
680
681 dst.check_usage(wgt::TextureUsages::COPY_DST)
682 .map_err(TransferError::MissingTextureUsage)?;
683
684 // Note: Doing the copy range validation early is important because ensures that the
685 // dimensions are not going to cause overflow in other parts of the validation.
686 let (hal_copy_size, array_layer_count) =
687 validate_texture_copy_range(&destination, &dst.desc, CopySide::Destination, size)?;
688
689 let (selector, dst_base) = extract_texture_selector(&destination, size, &dst)?;
690
691 if !dst_base.aspect.is_one() {
692 return Err(TransferError::CopyAspectNotOne.into());
693 }
694
695 if !conv::is_valid_copy_dst_texture_format(dst.desc.format, destination.aspect) {
696 return Err(TransferError::CopyToForbiddenTextureFormat {
697 format: dst.desc.format,
698 aspect: destination.aspect,
699 }
700 .into());
701 }
702
703 // Note: `_source_bytes_per_array_layer` is ignored since we
704 // have a staging copy, and it can have a different value.
705 let (required_bytes_in_copy, _source_bytes_per_array_layer) = validate_linear_texture_data(
706 data_layout,
707 dst.desc.format,
708 destination.aspect,
709 data.len() as wgt::BufferAddress,
710 CopySide::Source,
711 size,
712 false,
713 )?;
714
715 if dst.desc.format.is_depth_stencil_format() {
716 self.device
717 .require_downlevel_flags(wgt::DownlevelFlags::DEPTH_TEXTURE_AND_BUFFER_COPIES)
718 .map_err(TransferError::from)?;
719 }
720
721 let snatch_guard = self.device.snatchable_lock.read();
722
723 let mut pending_writes = self.pending_writes.lock();
724 let encoder = pending_writes.activate();
725
726 // If the copy does not fully cover the layers, we need to initialize to
727 // zero *first* as we don't keep track of partial texture layer inits.
728 //
729 // Strictly speaking we only need to clear the areas of a layer
730 // untouched, but this would get increasingly messy.
731 let init_layer_range = if dst.desc.dimension == wgt::TextureDimension::D3 {
732 // volume textures don't have a layer range as array volumes aren't supported
733 0..1
734 } else {
735 destination.origin.z..destination.origin.z + size.depth_or_array_layers
736 };
737 let mut dst_initialization_status = dst.initialization_status.write();
738 if dst_initialization_status.mips[destination.mip_level as usize]
739 .check(init_layer_range.clone())
740 .is_some()
741 {
742 if has_copy_partial_init_tracker_coverage(size, destination.mip_level, &dst.desc) {
743 for layer_range in dst_initialization_status.mips[destination.mip_level as usize]
744 .drain(init_layer_range)
745 .collect::<Vec<core::ops::Range<u32>>>()
746 {
747 let mut trackers = self.device.trackers.lock();
748 crate::command::clear_texture(
749 &dst,
750 TextureInitRange {
751 mip_range: destination.mip_level..(destination.mip_level + 1),
752 layer_range,
753 },
754 encoder,
755 &mut trackers.textures,
756 &self.device.alignments,
757 self.device.zero_buffer.as_ref(),
758 &snatch_guard,
759 )
760 .map_err(QueueWriteError::from)?;
761 }
762 } else {
763 dst_initialization_status.mips[destination.mip_level as usize]
764 .drain(init_layer_range);
765 }
766 }
767
768 let dst_raw = dst.try_raw(&snatch_guard)?;
769
770 let (block_width, block_height) = dst.desc.format.block_dimensions();
771 let width_in_blocks = size.width / block_width;
772 let height_in_blocks = size.height / block_height;
773
774 let block_size = dst
775 .desc
776 .format
777 .block_copy_size(Some(destination.aspect))
778 .unwrap();
779 let bytes_in_last_row = width_in_blocks * block_size;
780
781 let bytes_per_row = data_layout.bytes_per_row.unwrap_or(bytes_in_last_row);
782 let rows_per_image = data_layout.rows_per_image.unwrap_or(height_in_blocks);
783
784 let bytes_per_row_alignment = get_lowest_common_denom(
785 self.device.alignments.buffer_copy_pitch.get() as u32,
786 block_size,
787 );
788 let stage_bytes_per_row = wgt::math::align_to(bytes_in_last_row, bytes_per_row_alignment);
789
790 // Platform validation requires that the staging buffer always be
791 // freed, even if an error occurs. All paths from here must call
792 // `device.pending_writes.consume`.
793 let staging_buffer = if stage_bytes_per_row == bytes_per_row {
794 profiling::scope!("copy aligned");
795 // Fast path if the data is already being aligned optimally.
796 let stage_size = wgt::BufferSize::new(required_bytes_in_copy).unwrap();
797 let mut staging_buffer = StagingBuffer::new(&self.device, stage_size)?;
798 staging_buffer.write(&data[data_layout.offset as usize..]);
799 staging_buffer
800 } else {
801 profiling::scope!("copy chunked");
802 // Copy row by row into the optimal alignment.
803 let block_rows_in_copy =
804 (size.depth_or_array_layers - 1) * rows_per_image + height_in_blocks;
805 let stage_size =
806 wgt::BufferSize::new(stage_bytes_per_row as u64 * block_rows_in_copy as u64)
807 .unwrap();
808 let mut staging_buffer = StagingBuffer::new(&self.device, stage_size)?;
809 let copy_bytes_per_row = stage_bytes_per_row.min(bytes_per_row) as usize;
810 for layer in 0..size.depth_or_array_layers {
811 let rows_offset = layer * rows_per_image;
812 for row in rows_offset..rows_offset + height_in_blocks {
813 let src_offset = data_layout.offset as u32 + row * bytes_per_row;
814 let dst_offset = row * stage_bytes_per_row;
815 unsafe {
816 staging_buffer.write_with_offset(
817 data,
818 src_offset as isize,
819 dst_offset as isize,
820 copy_bytes_per_row,
821 )
822 }
823 }
824 }
825 staging_buffer
826 };
827
828 let staging_buffer = staging_buffer.flush();
829
830 let regions = (0..array_layer_count)
831 .map(|array_layer_offset| {
832 let mut texture_base = dst_base.clone();
833 texture_base.array_layer += array_layer_offset;
834 hal::BufferTextureCopy {
835 buffer_layout: wgt::TexelCopyBufferLayout {
836 offset: array_layer_offset as u64
837 * rows_per_image as u64
838 * stage_bytes_per_row as u64,
839 bytes_per_row: Some(stage_bytes_per_row),
840 rows_per_image: Some(rows_per_image),
841 },
842 texture_base,
843 size: hal_copy_size,
844 }
845 })
846 .collect::<Vec<_>>();
847
848 {
849 let buffer_barrier = hal::BufferBarrier {
850 buffer: staging_buffer.raw(),
851 usage: hal::StateTransition {
852 from: wgt::BufferUses::MAP_WRITE,
853 to: wgt::BufferUses::COPY_SRC,
854 },
855 };
856
857 let mut trackers = self.device.trackers.lock();
858 let transition =
859 trackers
860 .textures
861 .set_single(&dst, selector, wgt::TextureUses::COPY_DST);
862 let texture_barriers = transition
863 .map(|pending| pending.into_hal(dst_raw))
864 .collect::<Vec<_>>();
865
866 unsafe {
867 encoder.transition_textures(&texture_barriers);
868 encoder.transition_buffers(&[buffer_barrier]);
869 encoder.copy_buffer_to_texture(staging_buffer.raw(), dst_raw, &regions);
870 }
871 }
872
873 pending_writes.consume(staging_buffer);
874 pending_writes.insert_texture(&dst);
875
876 Ok(())
877 }
878
879 #[cfg(webgl)]
880 pub fn copy_external_image_to_texture(
881 &self,
882 source: &wgt::CopyExternalImageSourceInfo,
883 destination: wgt::CopyExternalImageDestInfo<Fallible<Texture>>,
884 size: wgt::Extent3d,
885 ) -> Result<(), QueueWriteError> {
886 profiling::scope!("Queue::copy_external_image_to_texture");
887
888 self.device.check_is_valid()?;
889
890 if size.width == 0 || size.height == 0 || size.depth_or_array_layers == 0 {
891 log::trace!("Ignoring write_texture of size 0");
892 return Ok(());
893 }
894
895 let mut needs_flag = false;
896 needs_flag |= matches!(source.source, wgt::ExternalImageSource::OffscreenCanvas(_));
897 needs_flag |= source.origin != wgt::Origin2d::ZERO;
898 needs_flag |= destination.color_space != wgt::PredefinedColorSpace::Srgb;
899 #[allow(clippy::bool_comparison)]
900 if matches!(source.source, wgt::ExternalImageSource::ImageBitmap(_)) {
901 needs_flag |= source.flip_y != false;
902 needs_flag |= destination.premultiplied_alpha != false;
903 }
904
905 if needs_flag {
906 self.device
907 .require_downlevel_flags(wgt::DownlevelFlags::UNRESTRICTED_EXTERNAL_TEXTURE_COPIES)
908 .map_err(TransferError::from)?;
909 }
910
911 let src_width = source.source.width();
912 let src_height = source.source.height();
913
914 let dst = destination.texture.get()?;
915 let premultiplied_alpha = destination.premultiplied_alpha;
916 let destination = wgt::TexelCopyTextureInfo {
917 texture: (),
918 mip_level: destination.mip_level,
919 origin: destination.origin,
920 aspect: destination.aspect,
921 };
922
923 if !conv::is_valid_external_image_copy_dst_texture_format(dst.desc.format) {
924 return Err(
925 TransferError::ExternalCopyToForbiddenTextureFormat(dst.desc.format).into(),
926 );
927 }
928 if dst.desc.dimension != wgt::TextureDimension::D2 {
929 return Err(TransferError::InvalidDimensionExternal.into());
930 }
931 dst.check_usage(wgt::TextureUsages::COPY_DST | wgt::TextureUsages::RENDER_ATTACHMENT)
932 .map_err(TransferError::MissingTextureUsage)?;
933 if dst.desc.sample_count != 1 {
934 return Err(TransferError::InvalidSampleCount {
935 sample_count: dst.desc.sample_count,
936 }
937 .into());
938 }
939
940 if source.origin.x + size.width > src_width {
941 return Err(TransferError::TextureOverrun {
942 start_offset: source.origin.x,
943 end_offset: source.origin.x + size.width,
944 texture_size: src_width,
945 dimension: crate::resource::TextureErrorDimension::X,
946 side: CopySide::Source,
947 }
948 .into());
949 }
950 if source.origin.y + size.height > src_height {
951 return Err(TransferError::TextureOverrun {
952 start_offset: source.origin.y,
953 end_offset: source.origin.y + size.height,
954 texture_size: src_height,
955 dimension: crate::resource::TextureErrorDimension::Y,
956 side: CopySide::Source,
957 }
958 .into());
959 }
960 if size.depth_or_array_layers != 1 {
961 return Err(TransferError::TextureOverrun {
962 start_offset: 0,
963 end_offset: size.depth_or_array_layers,
964 texture_size: 1,
965 dimension: crate::resource::TextureErrorDimension::Z,
966 side: CopySide::Source,
967 }
968 .into());
969 }
970
971 // Note: Doing the copy range validation early is important because ensures that the
972 // dimensions are not going to cause overflow in other parts of the validation.
973 let (hal_copy_size, _) =
974 validate_texture_copy_range(&destination, &dst.desc, CopySide::Destination, &size)?;
975
976 let (selector, dst_base) = extract_texture_selector(&destination, &size, &dst)?;
977
978 let mut pending_writes = self.pending_writes.lock();
979 let encoder = pending_writes.activate();
980
981 // If the copy does not fully cover the layers, we need to initialize to
982 // zero *first* as we don't keep track of partial texture layer inits.
983 //
984 // Strictly speaking we only need to clear the areas of a layer
985 // untouched, but this would get increasingly messy.
986 let init_layer_range = if dst.desc.dimension == wgt::TextureDimension::D3 {
987 // volume textures don't have a layer range as array volumes aren't supported
988 0..1
989 } else {
990 destination.origin.z..destination.origin.z + size.depth_or_array_layers
991 };
992 let mut dst_initialization_status = dst.initialization_status.write();
993 if dst_initialization_status.mips[destination.mip_level as usize]
994 .check(init_layer_range.clone())
995 .is_some()
996 {
997 if has_copy_partial_init_tracker_coverage(&size, destination.mip_level, &dst.desc) {
998 for layer_range in dst_initialization_status.mips[destination.mip_level as usize]
999 .drain(init_layer_range)
1000 .collect::<Vec<core::ops::Range<u32>>>()
1001 {
1002 let mut trackers = self.device.trackers.lock();
1003 crate::command::clear_texture(
1004 &dst,
1005 TextureInitRange {
1006 mip_range: destination.mip_level..(destination.mip_level + 1),
1007 layer_range,
1008 },
1009 encoder,
1010 &mut trackers.textures,
1011 &self.device.alignments,
1012 self.device.zero_buffer.as_ref(),
1013 &self.device.snatchable_lock.read(),
1014 )
1015 .map_err(QueueWriteError::from)?;
1016 }
1017 } else {
1018 dst_initialization_status.mips[destination.mip_level as usize]
1019 .drain(init_layer_range);
1020 }
1021 }
1022
1023 let snatch_guard = self.device.snatchable_lock.read();
1024 let dst_raw = dst.try_raw(&snatch_guard)?;
1025
1026 let regions = hal::TextureCopy {
1027 src_base: hal::TextureCopyBase {
1028 mip_level: 0,
1029 array_layer: 0,
1030 origin: source.origin.to_3d(0),
1031 aspect: hal::FormatAspects::COLOR,
1032 },
1033 dst_base,
1034 size: hal_copy_size,
1035 };
1036
1037 let mut trackers = self.device.trackers.lock();
1038 let transitions = trackers
1039 .textures
1040 .set_single(&dst, selector, wgt::TextureUses::COPY_DST);
1041
1042 // `copy_external_image_to_texture` is exclusive to the WebGL backend.
1043 // Don't go through the `DynCommandEncoder` abstraction and directly to the WebGL backend.
1044 let encoder_webgl = encoder
1045 .as_any_mut()
1046 .downcast_mut::<hal::gles::CommandEncoder>()
1047 .unwrap();
1048 let dst_raw_webgl = dst_raw
1049 .as_any()
1050 .downcast_ref::<hal::gles::Texture>()
1051 .unwrap();
1052 let transitions_webgl = transitions.map(|pending| {
1053 let dyn_transition = pending.into_hal(dst_raw);
1054 hal::TextureBarrier {
1055 texture: dst_raw_webgl,
1056 range: dyn_transition.range,
1057 usage: dyn_transition.usage,
1058 }
1059 });
1060
1061 use hal::CommandEncoder as _;
1062 unsafe {
1063 encoder_webgl.transition_textures(transitions_webgl);
1064 encoder_webgl.copy_external_image_to_texture(
1065 source,
1066 dst_raw_webgl,
1067 premultiplied_alpha,
1068 iter::once(regions),
1069 );
1070 }
1071
1072 Ok(())
1073 }
1074
1075 pub fn submit(
1076 &self,
1077 command_buffers: &[Arc<CommandBuffer>],
1078 ) -> Result<SubmissionIndex, (SubmissionIndex, QueueSubmitError)> {
1079 profiling::scope!("Queue::submit");
1080 api_log!("Queue::submit");
1081
1082 let submit_index;
1083
1084 let res = 'error: {
1085 let snatch_guard = self.device.snatchable_lock.read();
1086
1087 // Fence lock must be acquired after the snatch lock everywhere to avoid deadlocks.
1088 let mut fence = self.device.fence.write();
1089
1090 let mut command_index_guard = self.device.command_indices.write();
1091 command_index_guard.active_submission_index += 1;
1092 submit_index = command_index_guard.active_submission_index;
1093
1094 if let Err(e) = self.device.check_is_valid() {
1095 break 'error Err(e.into());
1096 }
1097
1098 let mut active_executions = Vec::new();
1099
1100 let mut used_surface_textures = track::TextureUsageScope::default();
1101
1102 // Use a hashmap here to deduplicate the surface textures that are used in the command buffers.
1103 // This avoids vulkan deadlocking from the same surface texture being submitted multiple times.
1104 let mut submit_surface_textures_owned = FastHashMap::default();
1105
1106 {
1107 if !command_buffers.is_empty() {
1108 profiling::scope!("prepare");
1109
1110 let mut first_error = None;
1111
1112 //TODO: if multiple command buffers are submitted, we can re-use the last
1113 // native command buffer of the previous chain instead of always creating
1114 // a temporary one, since the chains are not finished.
1115
1116 // finish all the command buffers first
1117 for command_buffer in command_buffers {
1118 profiling::scope!("process command buffer");
1119
1120 // we reset the used surface textures every time we use
1121 // it, so make sure to set_size on it.
1122 used_surface_textures.set_size(self.device.tracker_indices.textures.size());
1123
1124 // Note that we are required to invalidate all command buffers in both the success and failure paths.
1125 // This is why we `continue` and don't early return via `?`.
1126 #[allow(unused_mut)]
1127 let mut cmd_buf_data = command_buffer.take_finished();
1128
1129 #[cfg(feature = "trace")]
1130 if let Some(ref mut trace) = *self.device.trace.lock() {
1131 if let Ok(ref mut cmd_buf_data) = cmd_buf_data {
1132 trace.add(Action::Submit(
1133 submit_index,
1134 cmd_buf_data.commands.take().unwrap(),
1135 ));
1136 }
1137 }
1138
1139 if first_error.is_some() {
1140 continue;
1141 }
1142
1143 let mut baked = match cmd_buf_data {
1144 Ok(cmd_buf_data) => {
1145 let res = validate_command_buffer(
1146 command_buffer,
1147 self,
1148 &cmd_buf_data,
1149 &snatch_guard,
1150 &mut submit_surface_textures_owned,
1151 &mut used_surface_textures,
1152 &mut command_index_guard,
1153 );
1154 if let Err(err) = res {
1155 first_error.get_or_insert(err);
1156 continue;
1157 }
1158 cmd_buf_data.into_baked_commands()
1159 }
1160 Err(err) => {
1161 first_error.get_or_insert(err.into());
1162 continue;
1163 }
1164 };
1165
1166 // execute resource transitions
1167 if let Err(e) = baked.encoder.open_pass(Some("(wgpu internal) Transit")) {
1168 break 'error Err(e.into());
1169 }
1170
1171 //Note: locking the trackers has to be done after the storages
1172 let mut trackers = self.device.trackers.lock();
1173 if let Err(e) = baked.initialize_buffer_memory(&mut trackers, &snatch_guard)
1174 {
1175 break 'error Err(e.into());
1176 }
1177 if let Err(e) = baked.initialize_texture_memory(
1178 &mut trackers,
1179 &self.device,
1180 &snatch_guard,
1181 ) {
1182 break 'error Err(e.into());
1183 }
1184
1185 //Note: stateless trackers are not merged:
1186 // device already knows these resources exist.
1187 CommandBuffer::insert_barriers_from_device_tracker(
1188 baked.encoder.raw.as_mut(),
1189 &mut trackers,
1190 &baked.trackers,
1191 &snatch_guard,
1192 );
1193
1194 if let Err(e) = baked.encoder.close_and_push_front() {
1195 break 'error Err(e.into());
1196 }
1197
1198 // Transition surface textures into `Present` state.
1199 // Note: we could technically do it after all of the command buffers,
1200 // but here we have a command encoder by hand, so it's easier to use it.
1201 if !used_surface_textures.is_empty() {
1202 if let Err(e) = baked.encoder.open_pass(Some("(wgpu internal) Present"))
1203 {
1204 break 'error Err(e.into());
1205 }
1206 let texture_barriers = trackers
1207 .textures
1208 .set_from_usage_scope_and_drain_transitions(
1209 &used_surface_textures,
1210 &snatch_guard,
1211 )
1212 .collect::<Vec<_>>();
1213 unsafe {
1214 baked.encoder.raw.transition_textures(&texture_barriers);
1215 };
1216 if let Err(e) = baked.encoder.close() {
1217 break 'error Err(e.into());
1218 }
1219 used_surface_textures = track::TextureUsageScope::default();
1220 }
1221
1222 // done
1223 active_executions.push(EncoderInFlight {
1224 inner: baked.encoder,
1225 trackers: baked.trackers,
1226 temp_resources: baked.temp_resources,
1227 _indirect_draw_validation_resources: baked
1228 .indirect_draw_validation_resources,
1229 pending_buffers: FastHashMap::default(),
1230 pending_textures: FastHashMap::default(),
1231 });
1232 }
1233
1234 if let Some(first_error) = first_error {
1235 break 'error Err(first_error);
1236 }
1237 }
1238 }
1239
1240 let mut pending_writes = self.pending_writes.lock();
1241
1242 {
1243 used_surface_textures.set_size(self.device.tracker_indices.textures.size());
1244 for texture in pending_writes.dst_textures.values() {
1245 match texture.try_inner(&snatch_guard) {
1246 Ok(TextureInner::Native { .. }) => {}
1247 Ok(TextureInner::Surface { .. }) => {
1248 // Compare the Arcs by pointer as Textures don't implement Eq
1249 submit_surface_textures_owned
1250 .insert(Arc::as_ptr(texture), texture.clone());
1251
1252 unsafe {
1253 used_surface_textures
1254 .merge_single(texture, None, wgt::TextureUses::PRESENT)
1255 .unwrap()
1256 };
1257 }
1258 Err(e) => break 'error Err(e.into()),
1259 }
1260 }
1261
1262 if !used_surface_textures.is_empty() {
1263 let mut trackers = self.device.trackers.lock();
1264
1265 let texture_barriers = trackers
1266 .textures
1267 .set_from_usage_scope_and_drain_transitions(
1268 &used_surface_textures,
1269 &snatch_guard,
1270 )
1271 .collect::<Vec<_>>();
1272 unsafe {
1273 pending_writes
1274 .command_encoder
1275 .transition_textures(&texture_barriers);
1276 };
1277 }
1278 }
1279
1280 match pending_writes.pre_submit(&self.device.command_allocator, &self.device, self) {
1281 Ok(Some(pending_execution)) => {
1282 active_executions.insert(0, pending_execution);
1283 }
1284 Ok(None) => {}
1285 Err(e) => break 'error Err(e.into()),
1286 }
1287 let hal_command_buffers = active_executions
1288 .iter()
1289 .flat_map(|e| e.inner.list.iter().map(|b| b.as_ref()))
1290 .collect::<Vec<_>>();
1291
1292 {
1293 let mut submit_surface_textures =
1294 SmallVec::<[&dyn hal::DynSurfaceTexture; 2]>::with_capacity(
1295 submit_surface_textures_owned.len(),
1296 );
1297
1298 for texture in submit_surface_textures_owned.values() {
1299 let raw = match texture.inner.get(&snatch_guard) {
1300 Some(TextureInner::Surface { raw, .. }) => raw.as_ref(),
1301 _ => unreachable!(),
1302 };
1303 submit_surface_textures.push(raw);
1304 }
1305
1306 if let Err(e) = unsafe {
1307 self.raw().submit(
1308 &hal_command_buffers,
1309 &submit_surface_textures,
1310 (fence.as_mut(), submit_index),
1311 )
1312 }
1313 .map_err(|e| self.device.handle_hal_error(e))
1314 {
1315 break 'error Err(e.into());
1316 }
1317
1318 drop(command_index_guard);
1319
1320 // Advance the successful submission index.
1321 self.device
1322 .last_successful_submission_index
1323 .fetch_max(submit_index, Ordering::SeqCst);
1324 }
1325
1326 profiling::scope!("cleanup");
1327
1328 // this will register the new submission to the life time tracker
1329 self.lock_life()
1330 .track_submission(submit_index, active_executions);
1331 drop(pending_writes);
1332
1333 // This will schedule destruction of all resources that are no longer needed
1334 // by the user but used in the command stream, among other things.
1335 let fence_guard = RwLockWriteGuard::downgrade(fence);
1336 let (closures, result) =
1337 self.device
1338 .maintain(fence_guard, wgt::PollType::Poll, snatch_guard);
1339 match result {
1340 Ok(status) => {
1341 debug_assert!(matches!(
1342 status,
1343 wgt::PollStatus::QueueEmpty | wgt::PollStatus::Poll
1344 ));
1345 }
1346 Err(WaitIdleError::Device(err)) => break 'error Err(QueueSubmitError::Queue(err)),
1347 Err(WaitIdleError::WrongSubmissionIndex(..)) => {
1348 unreachable!("Cannot get WrongSubmissionIndex from Poll")
1349 }
1350 Err(WaitIdleError::Timeout) => unreachable!("Cannot get Timeout from Poll"),
1351 };
1352
1353 Ok(closures)
1354 };
1355
1356 let callbacks = match res {
1357 Ok(ok) => ok,
1358 Err(e) => return Err((submit_index, e)),
1359 };
1360
1361 // the closures should execute with nothing locked!
1362 callbacks.fire();
1363
1364 self.device.lose_if_oom();
1365
1366 api_log!("Queue::submit returned submit index {submit_index}");
1367
1368 Ok(submit_index)
1369 }
1370
1371 pub fn get_timestamp_period(&self) -> f32 {
1372 unsafe { self.raw().get_timestamp_period() }
1373 }
1374
1375 /// `closure` is guaranteed to be called.
1376 pub fn on_submitted_work_done(
1377 &self,
1378 closure: SubmittedWorkDoneClosure,
1379 ) -> Option<SubmissionIndex> {
1380 api_log!("Queue::on_submitted_work_done");
1381 //TODO: flush pending writes
1382 self.lock_life().add_work_done_closure(closure)
1383 }
1384 }
1385
1386 impl Global {
1387 pub fn queue_write_buffer(
1388 &self,
1389 queue_id: QueueId,
1390 buffer_id: id::BufferId,
1391 buffer_offset: wgt::BufferAddress,
1392 data: &[u8],
1393 ) -> Result<(), QueueWriteError> {
1394 let queue = self.hub.queues.get(queue_id);
1395
1396 #[cfg(feature = "trace")]
1397 if let Some(ref mut trace) = *queue.device.trace.lock() {
1398 let data_path = trace.make_binary("bin", data);
1399 trace.add(Action::WriteBuffer {
1400 id: buffer_id,
1401 data: data_path,
1402 range: buffer_offset..buffer_offset + data.len() as u64,
1403 queued: true,
1404 });
1405 }
1406
1407 let buffer = self.hub.buffers.get(buffer_id);
1408 queue.write_buffer(buffer, buffer_offset, data)
1409 }
1410
1411 pub fn queue_create_staging_buffer(
1412 &self,
1413 queue_id: QueueId,
1414 buffer_size: wgt::BufferSize,
1415 id_in: Option<id::StagingBufferId>,
1416 ) -> Result<(id::StagingBufferId, NonNull<u8>), QueueWriteError> {
1417 let queue = self.hub.queues.get(queue_id);
1418 let (staging_buffer, ptr) = queue.create_staging_buffer(buffer_size)?;
1419
1420 let fid = self.hub.staging_buffers.prepare(id_in);
1421 let id = fid.assign(staging_buffer);
1422
1423 Ok((id, ptr))
1424 }
1425
1426 pub fn queue_write_staging_buffer(
1427 &self,
1428 queue_id: QueueId,
1429 buffer_id: id::BufferId,
1430 buffer_offset: wgt::BufferAddress,
1431 staging_buffer_id: id::StagingBufferId,
1432 ) -> Result<(), QueueWriteError> {
1433 let queue = self.hub.queues.get(queue_id);
1434 let buffer = self.hub.buffers.get(buffer_id);
1435 let staging_buffer = self.hub.staging_buffers.remove(staging_buffer_id);
1436 queue.write_staging_buffer(buffer, buffer_offset, staging_buffer)
1437 }
1438
1439 pub fn queue_validate_write_buffer(
1440 &self,
1441 queue_id: QueueId,
1442 buffer_id: id::BufferId,
1443 buffer_offset: u64,
1444 buffer_size: wgt::BufferSize,
1445 ) -> Result<(), QueueWriteError> {
1446 let queue = self.hub.queues.get(queue_id);
1447 let buffer = self.hub.buffers.get(buffer_id);
1448 queue.validate_write_buffer(buffer, buffer_offset, buffer_size)
1449 }
1450
1451 pub fn queue_write_texture(
1452 &self,
1453 queue_id: QueueId,
1454 destination: &TexelCopyTextureInfo,
1455 data: &[u8],
1456 data_layout: &wgt::TexelCopyBufferLayout,
1457 size: &wgt::Extent3d,
1458 ) -> Result<(), QueueWriteError> {
1459 let queue = self.hub.queues.get(queue_id);
1460
1461 #[cfg(feature = "trace")]
1462 if let Some(ref mut trace) = *queue.device.trace.lock() {
1463 let data_path = trace.make_binary("bin", data);
1464 trace.add(Action::WriteTexture {
1465 to: *destination,
1466 data: data_path,
1467 layout: *data_layout,
1468 size: *size,
1469 });
1470 }
1471
1472 let destination = wgt::TexelCopyTextureInfo {
1473 texture: self.hub.textures.get(destination.texture),
1474 mip_level: destination.mip_level,
1475 origin: destination.origin,
1476 aspect: destination.aspect,
1477 };
1478 queue.write_texture(destination, data, data_layout, size)
1479 }
1480
1481 #[cfg(webgl)]
1482 pub fn queue_copy_external_image_to_texture(
1483 &self,
1484 queue_id: QueueId,
1485 source: &wgt::CopyExternalImageSourceInfo,
1486 destination: crate::command::CopyExternalImageDestInfo,
1487 size: wgt::Extent3d,
1488 ) -> Result<(), QueueWriteError> {
1489 let queue = self.hub.queues.get(queue_id);
1490 let destination = wgt::CopyExternalImageDestInfo {
1491 texture: self.hub.textures.get(destination.texture),
1492 mip_level: destination.mip_level,
1493 origin: destination.origin,
1494 aspect: destination.aspect,
1495 color_space: destination.color_space,
1496 premultiplied_alpha: destination.premultiplied_alpha,
1497 };
1498 queue.copy_external_image_to_texture(source, destination, size)
1499 }
1500
1501 pub fn queue_submit(
1502 &self,
1503 queue_id: QueueId,
1504 command_buffer_ids: &[id::CommandBufferId],
1505 ) -> Result<SubmissionIndex, (SubmissionIndex, QueueSubmitError)> {
1506 let queue = self.hub.queues.get(queue_id);
1507 let command_buffer_guard = self.hub.command_buffers.read();
1508 let command_buffers = command_buffer_ids
1509 .iter()
1510 .map(|id| command_buffer_guard.get(*id))
1511 .collect::<Vec<_>>();
1512 drop(command_buffer_guard);
1513 queue.submit(&command_buffers)
1514 }
1515
1516 pub fn queue_get_timestamp_period(&self, queue_id: QueueId) -> f32 {
1517 let queue = self.hub.queues.get(queue_id);
1518
1519 if queue.device.timestamp_normalizer.get().unwrap().enabled() {
1520 return 1.0;
1521 }
1522
1523 queue.get_timestamp_period()
1524 }
1525
1526 pub fn queue_on_submitted_work_done(
1527 &self,
1528 queue_id: QueueId,
1529 closure: SubmittedWorkDoneClosure,
1530 ) -> SubmissionIndex {
1531 api_log!("Queue::on_submitted_work_done {queue_id:?}");
1532
1533 //TODO: flush pending writes
1534 let queue = self.hub.queues.get(queue_id);
1535 let result = queue.on_submitted_work_done(closure);
1536 result.unwrap_or(0) // '0' means no wait is necessary
1537 }
1538 }
1539
1540 fn validate_command_buffer(
1541 command_buffer: &CommandBuffer,
1542 queue: &Queue,
1543 cmd_buf_data: &crate::command::CommandBufferMutable,
1544 snatch_guard: &SnatchGuard,
1545 submit_surface_textures_owned: &mut FastHashMap<*const Texture, Arc<Texture>>,
1546 used_surface_textures: &mut track::TextureUsageScope,
1547 command_index_guard: &mut RwLockWriteGuard<CommandIndices>,
1548 ) -> Result<(), QueueSubmitError> {
1549 command_buffer.same_device_as(queue)?;
1550
1551 {
1552 profiling::scope!("check resource state");
1553
1554 {
1555 profiling::scope!("buffers");
1556 for buffer in cmd_buf_data.trackers.buffers.used_resources() {
1557 buffer.check_destroyed(snatch_guard)?;
1558
1559 match *buffer.map_state.lock() {
1560 BufferMapState::Idle => (),
1561 _ => return Err(QueueSubmitError::BufferStillMapped(buffer.error_ident())),
1562 }
1563 }
1564 }
1565 {
1566 profiling::scope!("textures");
1567 for texture in cmd_buf_data.trackers.textures.used_resources() {
1568 let should_extend = match texture.try_inner(snatch_guard)? {
1569 TextureInner::Native { .. } => false,
1570 TextureInner::Surface { .. } => {
1571 // Compare the Arcs by pointer as Textures don't implement Eq.
1572 submit_surface_textures_owned.insert(Arc::as_ptr(texture), texture.clone());
1573
1574 true
1575 }
1576 };
1577 if should_extend {
1578 unsafe {
1579 used_surface_textures
1580 .merge_single(texture, None, wgt::TextureUses::PRESENT)
1581 .unwrap();
1582 };
1583 }
1584 }
1585 }
1586
1587 if let Err(e) =
1588 cmd_buf_data.validate_acceleration_structure_actions(snatch_guard, command_index_guard)
1589 {
1590 return Err(e.into());
1591 }
1592 }
1593 Ok(())
1594 }