| 1 |
// SPDX-License-Identifier: GPL-2.0 |
| 2 |
|
| 3 |
//! This module provides a wrapper for the C `struct request` type. |
| 4 |
//! |
| 5 |
//! C header: [`include/linux/blk-mq.h`](srctree/include/linux/blk-mq.h) |
| 6 |
|
| 7 |
use crate::{ |
| 8 |
bindings, |
| 9 |
block::mq::Operations, |
| 10 |
error::Result, |
| 11 |
sync::{ |
| 12 |
aref::{ARef, AlwaysRefCounted}, |
| 13 |
atomic::Relaxed, |
| 14 |
Refcount, |
| 15 |
}, |
| 16 |
types::Opaque, |
| 17 |
}; |
| 18 |
use core::{marker::PhantomData, ptr::NonNull}; |
| 19 |
|
| 20 |
/// A wrapper around a blk-mq [`struct request`]. This represents an IO request. |
| 21 |
/// |
| 22 |
/// # Implementation details |
| 23 |
/// |
| 24 |
/// There are four states for a request that the Rust bindings care about: |
| 25 |
/// |
| 26 |
/// 1. Request is owned by block layer (refcount 0). |
| 27 |
/// 2. Request is owned by driver but with zero [`ARef`]s in existence |
| 28 |
/// (refcount 1). |
| 29 |
/// 3. Request is owned by driver with exactly one [`ARef`] in existence |
| 30 |
/// (refcount 2). |
| 31 |
/// 4. Request is owned by driver with more than one [`ARef`] in existence |
| 32 |
/// (refcount > 2). |
| 33 |
/// |
| 34 |
/// |
| 35 |
/// We need to track 1 and 2 to ensure we fail tag to request conversions for |
| 36 |
/// requests that are not owned by the driver. |
| 37 |
/// |
| 38 |
/// We need to track 3 and 4 to ensure that it is safe to end the request and hand |
| 39 |
/// back ownership to the block layer. |
| 40 |
/// |
| 41 |
/// Note that the driver can still obtain new `ARef` even if there is no `ARef`s in existence by |
| 42 |
/// using `tag_to_rq`, hence the need to distinguish B and C. |
| 43 |
/// |
| 44 |
/// The states are tracked through the private `refcount` field of |
| 45 |
/// `RequestDataWrapper`. This structure lives in the private data area of the C |
| 46 |
/// [`struct request`]. |
| 47 |
/// |
| 48 |
/// # Invariants |
| 49 |
/// |
| 50 |
/// * `self.0` is a valid [`struct request`] created by the C portion of the |
| 51 |
/// kernel. |
| 52 |
/// * The private data area associated with this request must be an initialized |
| 53 |
/// and valid `RequestDataWrapper<T>`. |
| 54 |
/// * `self` is reference counted by atomic modification of |
| 55 |
/// `self.wrapper_ref().refcount()`. |
| 56 |
/// |
| 57 |
/// [`struct request`]: srctree/include/linux/blk-mq.h |
| 58 |
/// |
| 59 |
#[repr(transparent)] |
| 60 |
pub struct Request<T>(Opaque<bindings::request>, PhantomData<T>); |
| 61 |
|
| 62 |
impl<T: Operations> Request<T> { |
| 63 |
/// Create an [`ARef<Request>`] from a [`struct request`] pointer. |
| 64 |
/// |
| 65 |
/// # Safety |
| 66 |
/// |
| 67 |
/// * The caller must own a refcount on `ptr` that is transferred to the |
| 68 |
/// returned [`ARef`]. |
| 69 |
/// * The type invariants for [`Request`] must hold for the pointee of `ptr`. |
| 70 |
/// |
| 71 |
/// [`struct request`]: srctree/include/linux/blk-mq.h |
| 72 |
pub(crate) unsafe fn aref_from_raw(ptr: *mut bindings::request) -> ARef<Self> { |
| 73 |
// INVARIANT: By the safety requirements of this function, invariants are upheld. |
| 74 |
// SAFETY: By the safety requirement of this function, we own a |
| 75 |
// reference count that we can pass to `ARef`. |
| 76 |
unsafe { ARef::from_raw(NonNull::new_unchecked(ptr.cast())) } |
| 77 |
} |
| 78 |
|
| 79 |
/// Notify the block layer that a request is going to be processed now. |
| 80 |
/// |
| 81 |
/// The block layer uses this hook to do proper initializations such as |
| 82 |
/// starting the timeout timer. It is a requirement that block device |
| 83 |
/// drivers call this function when starting to process a request. |
| 84 |
/// |
| 85 |
/// # Safety |
| 86 |
/// |
| 87 |
/// The caller must have exclusive ownership of `self`, that is |
| 88 |
/// `self.wrapper_ref().refcount() == 2`. |
| 89 |
pub(crate) unsafe fn start_unchecked(this: &ARef<Self>) { |
| 90 |
// SAFETY: By type invariant, `self.0` is a valid `struct request` and |
| 91 |
// we have exclusive access. |
| 92 |
unsafe { bindings::blk_mq_start_request(this.0.get()) }; |
| 93 |
} |
| 94 |
|
| 95 |
/// Try to take exclusive ownership of `this` by dropping the refcount to 0. |
| 96 |
/// This fails if `this` is not the only [`ARef`] pointing to the underlying |
| 97 |
/// [`Request`]. |
| 98 |
/// |
| 99 |
/// If the operation is successful, [`Ok`] is returned with a pointer to the |
| 100 |
/// C [`struct request`]. If the operation fails, `this` is returned in the |
| 101 |
/// [`Err`] variant. |
| 102 |
/// |
| 103 |
/// [`struct request`]: srctree/include/linux/blk-mq.h |
| 104 |
fn try_set_end(this: ARef<Self>) -> Result<*mut bindings::request, ARef<Self>> { |
| 105 |
// To hand back the ownership, we need the current refcount to be 2. |
| 106 |
// Since we can race with `TagSet::tag_to_rq`, this needs to atomically reduce |
| 107 |
// refcount to 0. `Refcount` does not provide a way to do this, so use the underlying |
| 108 |
// atomics directly. |
| 109 |
if let Err(_old) = this |
| 110 |
.wrapper_ref() |
| 111 |
.refcount() |
| 112 |
.as_atomic() |
| 113 |
.cmpxchg(2, 0, Relaxed) |
| 114 |
{ |
| 115 |
return Err(this); |
| 116 |
} |
| 117 |
|
| 118 |
let request_ptr = this.0.get(); |
| 119 |
core::mem::forget(this); |
| 120 |
|
| 121 |
Ok(request_ptr) |
| 122 |
} |
| 123 |
|
| 124 |
/// Notify the block layer that the request has been completed without errors. |
| 125 |
/// |
| 126 |
/// This function will return [`Err`] if `this` is not the only [`ARef`] |
| 127 |
/// referencing the request. |
| 128 |
pub fn end_ok(this: ARef<Self>) -> Result<(), ARef<Self>> { |
| 129 |
let request_ptr = Self::try_set_end(this)?; |
| 130 |
|
| 131 |
// SAFETY: By type invariant, `this.0` was a valid `struct request`. The |
| 132 |
// success of the call to `try_set_end` guarantees that there are no |
| 133 |
// `ARef`s pointing to this request. Therefore it is safe to hand it |
| 134 |
// back to the block layer. |
| 135 |
unsafe { |
| 136 |
bindings::blk_mq_end_request( |
| 137 |
request_ptr, |
| 138 |
bindings::BLK_STS_OK as bindings::blk_status_t, |
| 139 |
) |
| 140 |
}; |
| 141 |
|
| 142 |
Ok(()) |
| 143 |
} |
| 144 |
|
| 145 |
/// Complete the request by scheduling `Operations::complete` for |
| 146 |
/// execution. |
| 147 |
/// |
| 148 |
/// The function may be scheduled locally, via SoftIRQ or remotely via IPMI. |
| 149 |
/// See `blk_mq_complete_request_remote` in [`blk-mq.c`] for details. |
| 150 |
/// |
| 151 |
/// [`blk-mq.c`]: srctree/block/blk-mq.c |
| 152 |
pub fn complete(this: ARef<Self>) { |
| 153 |
let ptr = ARef::into_raw(this).cast::<bindings::request>().as_ptr(); |
| 154 |
// SAFETY: By type invariant, `self.0` is a valid `struct request` |
| 155 |
if !unsafe { bindings::blk_mq_complete_request_remote(ptr) } { |
| 156 |
// SAFETY: We released a refcount above that we can reclaim here. |
| 157 |
let this = unsafe { Request::aref_from_raw(ptr) }; |
| 158 |
T::complete(this); |
| 159 |
} |
| 160 |
} |
| 161 |
|
| 162 |
/// Return a pointer to the [`RequestDataWrapper`] stored in the private area |
| 163 |
/// of the request structure. |
| 164 |
/// |
| 165 |
/// # Safety |
| 166 |
/// |
| 167 |
/// - `this` must point to a valid allocation of size at least size of |
| 168 |
/// [`Self`] plus size of [`RequestDataWrapper`]. |
| 169 |
pub(crate) unsafe fn wrapper_ptr(this: *mut Self) -> NonNull<RequestDataWrapper> { |
| 170 |
let request_ptr = this.cast::<bindings::request>(); |
| 171 |
// SAFETY: By safety requirements for this function, `this` is a |
| 172 |
// valid allocation. |
| 173 |
let wrapper_ptr = |
| 174 |
unsafe { bindings::blk_mq_rq_to_pdu(request_ptr).cast::<RequestDataWrapper>() }; |
| 175 |
// SAFETY: By C API contract, `wrapper_ptr` points to a valid allocation |
| 176 |
// and is not null. |
| 177 |
unsafe { NonNull::new_unchecked(wrapper_ptr) } |
| 178 |
} |
| 179 |
|
| 180 |
/// Return a reference to the [`RequestDataWrapper`] stored in the private |
| 181 |
/// area of the request structure. |
| 182 |
pub(crate) fn wrapper_ref(&self) -> &RequestDataWrapper { |
| 183 |
// SAFETY: By type invariant, `self.0` is a valid allocation. Further, |
| 184 |
// the private data associated with this request is initialized and |
| 185 |
// valid. The existence of `&self` guarantees that the private data is |
| 186 |
// valid as a shared reference. |
| 187 |
unsafe { Self::wrapper_ptr(core::ptr::from_ref(self).cast_mut()).as_ref() } |
| 188 |
} |
| 189 |
} |
| 190 |
|
| 191 |
/// A wrapper around data stored in the private area of the C [`struct request`]. |
| 192 |
/// |
| 193 |
/// [`struct request`]: srctree/include/linux/blk-mq.h |
| 194 |
pub(crate) struct RequestDataWrapper { |
| 195 |
/// The Rust request refcount has the following states: |
| 196 |
/// |
| 197 |
/// - 0: The request is owned by C block layer. |
| 198 |
/// - 1: The request is owned by Rust abstractions but there are no [`ARef`] references to it. |
| 199 |
/// - 2+: There are [`ARef`] references to the request. |
| 200 |
refcount: Refcount, |
| 201 |
} |
| 202 |
|
| 203 |
impl RequestDataWrapper { |
| 204 |
/// Return a reference to the refcount of the request that is embedding |
| 205 |
/// `self`. |
| 206 |
pub(crate) fn refcount(&self) -> &Refcount { |
| 207 |
&self.refcount |
| 208 |
} |
| 209 |
|
| 210 |
/// Return a pointer to the refcount of the request that is embedding the |
| 211 |
/// pointee of `this`. |
| 212 |
/// |
| 213 |
/// # Safety |
| 214 |
/// |
| 215 |
/// - `this` must point to a live allocation of at least the size of `Self`. |
| 216 |
pub(crate) unsafe fn refcount_ptr(this: *mut Self) -> *mut Refcount { |
| 217 |
// SAFETY: Because of the safety requirements of this function, the |
| 218 |
// field projection is safe. |
| 219 |
unsafe { &raw mut (*this).refcount } |
| 220 |
} |
| 221 |
} |
| 222 |
|
| 223 |
// SAFETY: Exclusive access is thread-safe for `Request`. `Request` has no `&mut |
| 224 |
// self` methods and `&self` methods that mutate `self` are internally |
| 225 |
// synchronized. |
| 226 |
unsafe impl<T: Operations> Send for Request<T> {} |
| 227 |
|
| 228 |
// SAFETY: Shared access is thread-safe for `Request`. `&self` methods that |
| 229 |
// mutate `self` are internally synchronized` |
| 230 |
unsafe impl<T: Operations> Sync for Request<T> {} |
| 231 |
|
| 232 |
// SAFETY: All instances of `Request<T>` are reference counted. This |
| 233 |
// implementation of `AlwaysRefCounted` ensure that increments to the ref count |
| 234 |
// keeps the object alive in memory at least until a matching reference count |
| 235 |
// decrement is executed. |
| 236 |
unsafe impl<T: Operations> AlwaysRefCounted for Request<T> { |
| 237 |
fn inc_ref(&self) { |
| 238 |
self.wrapper_ref().refcount().inc(); |
| 239 |
} |
| 240 |
|
| 241 |
unsafe fn dec_ref(obj: core::ptr::NonNull<Self>) { |
| 242 |
// SAFETY: The type invariants of `ARef` guarantee that `obj` is valid |
| 243 |
// for read. |
| 244 |
let wrapper_ptr = unsafe { Self::wrapper_ptr(obj.as_ptr()).as_ptr() }; |
| 245 |
// SAFETY: The type invariant of `Request` guarantees that the private |
| 246 |
// data area is initialized and valid. |
| 247 |
let refcount = unsafe { &*RequestDataWrapper::refcount_ptr(wrapper_ptr) }; |
| 248 |
|
| 249 |
#[cfg_attr(not(CONFIG_DEBUG_MISC), allow(unused_variables))] |
| 250 |
let is_zero = refcount.dec_and_test(); |
| 251 |
|
| 252 |
#[cfg(CONFIG_DEBUG_MISC)] |
| 253 |
if is_zero { |
| 254 |
panic!("Request reached refcount zero in Rust abstractions"); |
| 255 |
} |
| 256 |
} |
| 257 |
} |