| 1 |
// SPDX-License-Identifier: GPL-2.0 |
| 2 |
|
| 3 |
//! This module provides an interface for blk-mq drivers to implement. |
| 4 |
//! |
| 5 |
//! C header: [`include/linux/blk-mq.h`](srctree/include/linux/blk-mq.h) |
| 6 |
|
| 7 |
use crate::{ |
| 8 |
bindings, |
| 9 |
block::mq::{request::RequestDataWrapper, Request}, |
| 10 |
error::{from_result, Result}, |
| 11 |
prelude::*, |
| 12 |
sync::{aref::ARef, Refcount}, |
| 13 |
types::ForeignOwnable, |
| 14 |
}; |
| 15 |
use core::marker::PhantomData; |
| 16 |
|
| 17 |
type ForeignBorrowed<'a, T> = <T as ForeignOwnable>::Borrowed<'a>; |
| 18 |
|
| 19 |
/// Implement this trait to interface blk-mq as block devices. |
| 20 |
/// |
| 21 |
/// To implement a block device driver, implement this trait as described in the |
| 22 |
/// [module level documentation]. The kernel will use the implementation of the |
| 23 |
/// functions defined in this trait to interface a block device driver. Note: |
| 24 |
/// There is no need for an exit_request() implementation, because the `drop` |
| 25 |
/// implementation of the [`Request`] type will be invoked by automatically by |
| 26 |
/// the C/Rust glue logic. |
| 27 |
/// |
| 28 |
/// [module level documentation]: kernel::block::mq |
| 29 |
#[macros::vtable] |
| 30 |
pub trait Operations: Sized { |
| 31 |
/// Data associated with the `struct request_queue` that is allocated for |
| 32 |
/// the `GenDisk` associated with this `Operations` implementation. |
| 33 |
type QueueData: ForeignOwnable; |
| 34 |
|
| 35 |
/// Called by the kernel to queue a request with the driver. If `is_last` is |
| 36 |
/// `false`, the driver is allowed to defer committing the request. |
| 37 |
fn queue_rq( |
| 38 |
queue_data: ForeignBorrowed<'_, Self::QueueData>, |
| 39 |
rq: ARef<Request<Self>>, |
| 40 |
is_last: bool, |
| 41 |
) -> Result; |
| 42 |
|
| 43 |
/// Called by the kernel to indicate that queued requests should be submitted. |
| 44 |
fn commit_rqs(queue_data: ForeignBorrowed<'_, Self::QueueData>); |
| 45 |
|
| 46 |
/// Called by the kernel when the request is completed. |
| 47 |
fn complete(rq: ARef<Request<Self>>); |
| 48 |
|
| 49 |
/// Called by the kernel to poll the device for completed requests. Only |
| 50 |
/// used for poll queues. |
| 51 |
fn poll() -> bool { |
| 52 |
build_error!(crate::error::VTABLE_DEFAULT_ERROR) |
| 53 |
} |
| 54 |
} |
| 55 |
|
| 56 |
/// A vtable for blk-mq to interact with a block device driver. |
| 57 |
/// |
| 58 |
/// A `bindings::blk_mq_ops` vtable is constructed from pointers to the `extern |
| 59 |
/// "C"` functions of this struct, exposed through the `OperationsVTable::VTABLE`. |
| 60 |
/// |
| 61 |
/// For general documentation of these methods, see the kernel source |
| 62 |
/// documentation related to `struct blk_mq_operations` in |
| 63 |
/// [`include/linux/blk-mq.h`]. |
| 64 |
/// |
| 65 |
/// [`include/linux/blk-mq.h`]: srctree/include/linux/blk-mq.h |
| 66 |
pub(crate) struct OperationsVTable<T: Operations>(PhantomData<T>); |
| 67 |
|
| 68 |
impl<T: Operations> OperationsVTable<T> { |
| 69 |
/// This function is called by the C kernel. A pointer to this function is |
| 70 |
/// installed in the `blk_mq_ops` vtable for the driver. |
| 71 |
/// |
| 72 |
/// # Safety |
| 73 |
/// |
| 74 |
/// - The caller of this function must ensure that the pointee of `bd` is |
| 75 |
/// valid for reads for the duration of this function. |
| 76 |
/// - This function must be called for an initialized and live `hctx`. That |
| 77 |
/// is, `Self::init_hctx_callback` was called and |
| 78 |
/// `Self::exit_hctx_callback()` was not yet called. |
| 79 |
/// - `(*bd).rq` must point to an initialized and live `bindings:request`. |
| 80 |
/// That is, `Self::init_request_callback` was called but |
| 81 |
/// `Self::exit_request_callback` was not yet called for the request. |
| 82 |
/// - `(*bd).rq` must be owned by the driver. That is, the block layer must |
| 83 |
/// promise to not access the request until the driver calls |
| 84 |
/// `bindings::blk_mq_end_request` for the request. |
| 85 |
unsafe extern "C" fn queue_rq_callback( |
| 86 |
hctx: *mut bindings::blk_mq_hw_ctx, |
| 87 |
bd: *const bindings::blk_mq_queue_data, |
| 88 |
) -> bindings::blk_status_t { |
| 89 |
// SAFETY: `bd.rq` is valid as required by the safety requirement for |
| 90 |
// this function. |
| 91 |
let request = unsafe { &*(*bd).rq.cast::<Request<T>>() }; |
| 92 |
|
| 93 |
// One refcount for the ARef, one for being in flight |
| 94 |
request.wrapper_ref().refcount().set(2); |
| 95 |
|
| 96 |
// SAFETY: |
| 97 |
// - We own a refcount that we took above. We pass that to `ARef`. |
| 98 |
// - By the safety requirements of this function, `request` is a valid |
| 99 |
// `struct request` and the private data is properly initialized. |
| 100 |
// - `rq` will be alive until `blk_mq_end_request` is called and is |
| 101 |
// reference counted by `ARef` until then. |
| 102 |
let rq = unsafe { Request::aref_from_raw((*bd).rq) }; |
| 103 |
|
| 104 |
// SAFETY: `hctx` is valid as required by this function. |
| 105 |
let queue_data = unsafe { (*(*hctx).queue).queuedata }; |
| 106 |
|
| 107 |
// SAFETY: `queue.queuedata` was created by `GenDiskBuilder::build` with |
| 108 |
// a call to `ForeignOwnable::into_foreign` to create `queuedata`. |
| 109 |
// `ForeignOwnable::from_foreign` is only called when the tagset is |
| 110 |
// dropped, which happens after we are dropped. |
| 111 |
let queue_data = unsafe { T::QueueData::borrow(queue_data) }; |
| 112 |
|
| 113 |
// SAFETY: We have exclusive access and we just set the refcount above. |
| 114 |
unsafe { Request::start_unchecked(&rq) }; |
| 115 |
|
| 116 |
let ret = T::queue_rq( |
| 117 |
queue_data, |
| 118 |
rq, |
| 119 |
// SAFETY: `bd` is valid as required by the safety requirement for |
| 120 |
// this function. |
| 121 |
unsafe { (*bd).last }, |
| 122 |
); |
| 123 |
|
| 124 |
if let Err(e) = ret { |
| 125 |
e.to_blk_status() |
| 126 |
} else { |
| 127 |
bindings::BLK_STS_OK as bindings::blk_status_t |
| 128 |
} |
| 129 |
} |
| 130 |
|
| 131 |
/// This function is called by the C kernel. A pointer to this function is |
| 132 |
/// installed in the `blk_mq_ops` vtable for the driver. |
| 133 |
/// |
| 134 |
/// # Safety |
| 135 |
/// |
| 136 |
/// This function may only be called by blk-mq C infrastructure. The caller |
| 137 |
/// must ensure that `hctx` is valid. |
| 138 |
unsafe extern "C" fn commit_rqs_callback(hctx: *mut bindings::blk_mq_hw_ctx) { |
| 139 |
// SAFETY: `hctx` is valid as required by this function. |
| 140 |
let queue_data = unsafe { (*(*hctx).queue).queuedata }; |
| 141 |
|
| 142 |
// SAFETY: `queue.queuedata` was created by `GenDisk::try_new()` with a |
| 143 |
// call to `ForeignOwnable::into_foreign()` to create `queuedata`. |
| 144 |
// `ForeignOwnable::from_foreign()` is only called when the tagset is |
| 145 |
// dropped, which happens after we are dropped. |
| 146 |
let queue_data = unsafe { T::QueueData::borrow(queue_data) }; |
| 147 |
T::commit_rqs(queue_data) |
| 148 |
} |
| 149 |
|
| 150 |
/// This function is called by the C kernel. A pointer to this function is |
| 151 |
/// installed in the `blk_mq_ops` vtable for the driver. |
| 152 |
/// |
| 153 |
/// # Safety |
| 154 |
/// |
| 155 |
/// This function may only be called by blk-mq C infrastructure. `rq` must |
| 156 |
/// point to a valid request that has been marked as completed. The pointee |
| 157 |
/// of `rq` must be valid for write for the duration of this function. |
| 158 |
unsafe extern "C" fn complete_callback(rq: *mut bindings::request) { |
| 159 |
// SAFETY: This function can only be dispatched through |
| 160 |
// `Request::complete`. We leaked a refcount then which we pick back up |
| 161 |
// now. |
| 162 |
let aref = unsafe { Request::aref_from_raw(rq) }; |
| 163 |
T::complete(aref); |
| 164 |
} |
| 165 |
|
| 166 |
/// This function is called by the C kernel. A pointer to this function is |
| 167 |
/// installed in the `blk_mq_ops` vtable for the driver. |
| 168 |
/// |
| 169 |
/// # Safety |
| 170 |
/// |
| 171 |
/// This function may only be called by blk-mq C infrastructure. |
| 172 |
unsafe extern "C" fn poll_callback( |
| 173 |
_hctx: *mut bindings::blk_mq_hw_ctx, |
| 174 |
_iob: *mut bindings::io_comp_batch, |
| 175 |
) -> crate::ffi::c_int { |
| 176 |
T::poll().into() |
| 177 |
} |
| 178 |
|
| 179 |
/// This function is called by the C kernel. A pointer to this function is |
| 180 |
/// installed in the `blk_mq_ops` vtable for the driver. |
| 181 |
/// |
| 182 |
/// # Safety |
| 183 |
/// |
| 184 |
/// This function may only be called by blk-mq C infrastructure. This |
| 185 |
/// function may only be called once before `exit_hctx_callback` is called |
| 186 |
/// for the same context. |
| 187 |
unsafe extern "C" fn init_hctx_callback( |
| 188 |
_hctx: *mut bindings::blk_mq_hw_ctx, |
| 189 |
_tagset_data: *mut crate::ffi::c_void, |
| 190 |
_hctx_idx: crate::ffi::c_uint, |
| 191 |
) -> crate::ffi::c_int { |
| 192 |
from_result(|| Ok(0)) |
| 193 |
} |
| 194 |
|
| 195 |
/// This function is called by the C kernel. A pointer to this function is |
| 196 |
/// installed in the `blk_mq_ops` vtable for the driver. |
| 197 |
/// |
| 198 |
/// # Safety |
| 199 |
/// |
| 200 |
/// This function may only be called by blk-mq C infrastructure. |
| 201 |
unsafe extern "C" fn exit_hctx_callback( |
| 202 |
_hctx: *mut bindings::blk_mq_hw_ctx, |
| 203 |
_hctx_idx: crate::ffi::c_uint, |
| 204 |
) { |
| 205 |
} |
| 206 |
|
| 207 |
/// This function is called by the C kernel. A pointer to this function is |
| 208 |
/// installed in the `blk_mq_ops` vtable for the driver. |
| 209 |
/// |
| 210 |
/// # Safety |
| 211 |
/// |
| 212 |
/// - This function may only be called by blk-mq C infrastructure. |
| 213 |
/// - `_set` must point to an initialized `TagSet<T>`. |
| 214 |
/// - `rq` must point to an initialized `bindings::request`. |
| 215 |
/// - The allocation pointed to by `rq` must be at the size of `Request` |
| 216 |
/// plus the size of `RequestDataWrapper`. |
| 217 |
unsafe extern "C" fn init_request_callback( |
| 218 |
_set: *mut bindings::blk_mq_tag_set, |
| 219 |
rq: *mut bindings::request, |
| 220 |
_hctx_idx: crate::ffi::c_uint, |
| 221 |
_numa_node: crate::ffi::c_uint, |
| 222 |
) -> crate::ffi::c_int { |
| 223 |
from_result(|| { |
| 224 |
// SAFETY: By the safety requirements of this function, `rq` points |
| 225 |
// to a valid allocation. |
| 226 |
let pdu = unsafe { Request::wrapper_ptr(rq.cast::<Request<T>>()) }; |
| 227 |
|
| 228 |
// SAFETY: The refcount field is allocated but not initialized, so |
| 229 |
// it is valid for writes. |
| 230 |
unsafe { RequestDataWrapper::refcount_ptr(pdu.as_ptr()).write(Refcount::new(0)) }; |
| 231 |
|
| 232 |
Ok(0) |
| 233 |
}) |
| 234 |
} |
| 235 |
|
| 236 |
/// This function is called by the C kernel. A pointer to this function is |
| 237 |
/// installed in the `blk_mq_ops` vtable for the driver. |
| 238 |
/// |
| 239 |
/// # Safety |
| 240 |
/// |
| 241 |
/// - This function may only be called by blk-mq C infrastructure. |
| 242 |
/// - `_set` must point to an initialized `TagSet<T>`. |
| 243 |
/// - `rq` must point to an initialized and valid `Request`. |
| 244 |
unsafe extern "C" fn exit_request_callback( |
| 245 |
_set: *mut bindings::blk_mq_tag_set, |
| 246 |
rq: *mut bindings::request, |
| 247 |
_hctx_idx: crate::ffi::c_uint, |
| 248 |
) { |
| 249 |
// SAFETY: The tagset invariants guarantee that all requests are allocated with extra memory |
| 250 |
// for the request data. |
| 251 |
let pdu = unsafe { bindings::blk_mq_rq_to_pdu(rq) }.cast::<RequestDataWrapper>(); |
| 252 |
|
| 253 |
// SAFETY: `pdu` is valid for read and write and is properly initialised. |
| 254 |
unsafe { core::ptr::drop_in_place(pdu) }; |
| 255 |
} |
| 256 |
|
| 257 |
const VTABLE: bindings::blk_mq_ops = bindings::blk_mq_ops { |
| 258 |
queue_rq: Some(Self::queue_rq_callback), |
| 259 |
queue_rqs: None, |
| 260 |
commit_rqs: Some(Self::commit_rqs_callback), |
| 261 |
get_budget: None, |
| 262 |
put_budget: None, |
| 263 |
set_rq_budget_token: None, |
| 264 |
get_rq_budget_token: None, |
| 265 |
timeout: None, |
| 266 |
poll: if T::HAS_POLL { |
| 267 |
Some(Self::poll_callback) |
| 268 |
} else { |
| 269 |
None |
| 270 |
}, |
| 271 |
complete: Some(Self::complete_callback), |
| 272 |
init_hctx: Some(Self::init_hctx_callback), |
| 273 |
exit_hctx: Some(Self::exit_hctx_callback), |
| 274 |
init_request: Some(Self::init_request_callback), |
| 275 |
exit_request: Some(Self::exit_request_callback), |
| 276 |
cleanup_rq: None, |
| 277 |
busy: None, |
| 278 |
map_queues: None, |
| 279 |
#[cfg(CONFIG_BLK_DEBUG_FS)] |
| 280 |
show_rq: None, |
| 281 |
}; |
| 282 |
|
| 283 |
pub(crate) const fn build() -> &'static bindings::blk_mq_ops { |
| 284 |
&Self::VTABLE |
| 285 |
} |
| 286 |
} |