| 1 |
// Copyright © 2024 Collabora, Ltd. |
| 2 |
// SPDX-License-Identifier: MIT |
| 3 |
|
| 4 |
use nvidia_headers::ArrayMthd; |
| 5 |
use nvidia_headers::Mthd; |
| 6 |
|
| 7 |
pub const MAX_MTHD_LEN: u16 = 0x1fff; |
| 8 |
pub const MAX_MTHD_ADDR: u16 = 0x7fff; |
| 9 |
|
| 10 |
fn class_to_subc(class: u16) -> u8 { |
| 11 |
match class & 0xff { |
| 12 |
0x97 => 0, |
| 13 |
0xc0 => 1, |
| 14 |
0x39 => 2, |
| 15 |
0x2d => 3, |
| 16 |
0xb5 => 4, |
| 17 |
0xb0 => 4, |
| 18 |
_ => panic!("Invalid class: {class}"), |
| 19 |
} |
| 20 |
} |
| 21 |
|
| 22 |
#[repr(u8)] |
| 23 |
enum MthdType { |
| 24 |
/// Each dword increments the address by one |
| 25 |
NInc = 1, |
| 26 |
/// The first dword increments the address by one |
| 27 |
OneInc = 3, |
| 28 |
/// Instead of a length, stores 13 bits of immediate data |
| 29 |
Immd = 4, |
| 30 |
/// The address is not incremented |
| 31 |
ZeroInc = 5, |
| 32 |
} |
| 33 |
|
| 34 |
impl TryFrom<u8> for MthdType { |
| 35 |
type Error = &'static str; |
| 36 |
|
| 37 |
fn try_from(value: u8) -> Result<Self, Self::Error> { |
| 38 |
match value { |
| 39 |
1 => Ok(MthdType::NInc), |
| 40 |
3 => Ok(MthdType::OneInc), |
| 41 |
4 => Ok(MthdType::Immd), |
| 42 |
5 => Ok(MthdType::ZeroInc), |
| 43 |
_ => Err("Invalid method type"), |
| 44 |
} |
| 45 |
} |
| 46 |
} |
| 47 |
|
| 48 |
/// A method header. |
| 49 |
/// |
| 50 |
/// Methods start with a header that can encode the `IncType`, the subclass, |
| 51 |
/// an address and the size. Optionally, the header can contain an address |
| 52 |
/// and an immediate instead. |
| 53 |
#[repr(transparent)] |
| 54 |
struct MthdHeader(u32); |
| 55 |
|
| 56 |
impl MthdHeader { |
| 57 |
fn from_bits_mut(bits: &mut u32) -> &mut Self { |
| 58 |
// This is always safe beause a reference is always safe to |
| 59 |
// derefence. |
| 60 |
unsafe { &mut *(bits as *mut u32 as *mut MthdHeader) } |
| 61 |
} |
| 62 |
|
| 63 |
fn into_bits(self) -> u32 { |
| 64 |
self.0 |
| 65 |
} |
| 66 |
|
| 67 |
fn new(mthd_type: MthdType, subc: u8, addr: u16, data: u16) -> Self { |
| 68 |
debug_assert!(subc <= 0x7); |
| 69 |
debug_assert!(addr & 0x3 == 0 && addr <= MAX_MTHD_ADDR); |
| 70 |
debug_assert!(data <= MAX_MTHD_LEN); |
| 71 |
|
| 72 |
let mthd_type: u32 = (mthd_type as u8).into(); |
| 73 |
let subc: u32 = subc.into(); |
| 74 |
let addr: u32 = addr.into(); |
| 75 |
let data: u32 = data.into(); |
| 76 |
|
| 77 |
Self((mthd_type << 29) | (data << 16) | (subc << 13) | (addr >> 2)) |
| 78 |
} |
| 79 |
|
| 80 |
fn new_immd(immd: u16, subc: u8, addr: u16) -> Self { |
| 81 |
Self::new(MthdType::Immd, subc, addr, immd) |
| 82 |
} |
| 83 |
|
| 84 |
fn mthd_type(&self) -> MthdType { |
| 85 |
((self.0 >> 29) as u8).try_into().unwrap() |
| 86 |
} |
| 87 |
|
| 88 |
fn set_mthd_type(&mut self, mthd_type: MthdType) { |
| 89 |
self.0 &= 0x1fffffff; |
| 90 |
self.0 |= (mthd_type as u8 as u32) << 29; |
| 91 |
} |
| 92 |
|
| 93 |
fn subc(&self) -> u8 { |
| 94 |
((self.0 >> 13) & 0x7) as u8 |
| 95 |
} |
| 96 |
|
| 97 |
fn addr(&self) -> u16 { |
| 98 |
((self.0 & 0x1fff) << 2) as u16 |
| 99 |
} |
| 100 |
|
| 101 |
fn len(&self) -> u16 { |
| 102 |
debug_assert!(!matches!(self.mthd_type(), MthdType::Immd)); |
| 103 |
((self.0 >> 16) & 0x1fff) as u16 |
| 104 |
} |
| 105 |
|
| 106 |
fn set_len(&mut self, len: u16) { |
| 107 |
debug_assert!(len <= MAX_MTHD_LEN); |
| 108 |
self.0 &= 0xe000ffff; |
| 109 |
self.0 |= u32::from(len) << 16; |
| 110 |
} |
| 111 |
|
| 112 |
fn add_len(&mut self, count: u16) { |
| 113 |
let new_len = self.len() + count; |
| 114 |
self.set_len(new_len); |
| 115 |
} |
| 116 |
} |
| 117 |
|
| 118 |
pub struct Push { |
| 119 |
/// The internal memory. Has to be uploaded to a BO through flush(). |
| 120 |
mem: Vec<u32>, |
| 121 |
/// Last DW that is an incrementing type or usize::MAX |
| 122 |
last_inc: usize, |
| 123 |
} |
| 124 |
|
| 125 |
impl Push { |
| 126 |
/// Instantiates a new push buffer. |
| 127 |
pub fn new() -> Self { |
| 128 |
Self { |
| 129 |
mem: Vec::new(), |
| 130 |
last_inc: usize::MAX, |
| 131 |
} |
| 132 |
} |
| 133 |
|
| 134 |
fn push_mthd_bits(&mut self, subc: u8, addr: u16, bits: u32) { |
| 135 |
let current_len = self.mem.len(); |
| 136 |
if let Some(last) = self.mem.get_mut(self.last_inc) { |
| 137 |
let last = MthdHeader::from_bits_mut(last); |
| 138 |
debug_assert!(last.len() >= 1); |
| 139 |
debug_assert!( |
| 140 |
self.last_inc + usize::from(last.len()) + 1 == current_len |
| 141 |
); |
| 142 |
if subc == last.subc() { |
| 143 |
match last.mthd_type() { |
| 144 |
MthdType::NInc => { |
| 145 |
if addr == last.addr() + last.len() * 4 { |
| 146 |
last.add_len(1); |
| 147 |
self.mem.push(bits); |
| 148 |
return; |
| 149 |
} else if last.len() == 1 && addr == last.addr() { |
| 150 |
last.set_mthd_type(MthdType::ZeroInc); |
| 151 |
last.add_len(1); |
| 152 |
self.mem.push(bits); |
| 153 |
return; |
| 154 |
} else if last.len() == 2 && addr == last.addr() + 4 { |
| 155 |
last.set_mthd_type(MthdType::OneInc); |
| 156 |
last.add_len(1); |
| 157 |
self.mem.push(bits); |
| 158 |
return; |
| 159 |
} |
| 160 |
} |
| 161 |
MthdType::ZeroInc => { |
| 162 |
if addr == last.addr() { |
| 163 |
last.add_len(1); |
| 164 |
self.mem.push(bits); |
| 165 |
return; |
| 166 |
} |
| 167 |
} |
| 168 |
MthdType::OneInc => { |
| 169 |
if addr == last.addr() + 4 { |
| 170 |
last.add_len(1); |
| 171 |
self.mem.push(bits); |
| 172 |
return; |
| 173 |
} |
| 174 |
} |
| 175 |
_ => (), |
| 176 |
} |
| 177 |
} |
| 178 |
} |
| 179 |
|
| 180 |
// Otherwise, we need a new method header. |
| 181 |
// |
| 182 |
// Methods that use 13bits or lower can be encoded as immediates |
| 183 |
// directly. |
| 184 |
if bits <= 0x1fff { |
| 185 |
self.last_inc = usize::MAX; |
| 186 |
let header = MthdHeader::new_immd(bits as u16, subc, addr); |
| 187 |
self.mem.push(header.into_bits()); |
| 188 |
} else { |
| 189 |
self.last_inc = self.mem.len(); |
| 190 |
let header = MthdHeader::new(MthdType::NInc, subc, addr, 1); |
| 191 |
self.mem.push(header.into_bits()); |
| 192 |
self.mem.push(bits); |
| 193 |
} |
| 194 |
} |
| 195 |
|
| 196 |
pub fn push_method<M: Mthd>(&mut self, mthd: M) { |
| 197 |
self.push_mthd_bits(class_to_subc(M::CLASS), M::ADDR, mthd.to_bits()); |
| 198 |
} |
| 199 |
|
| 200 |
pub fn push_mthd_0inc<M: Mthd>(&mut self, mthd: M) { |
| 201 |
assert!(mthd.to_bits() == 0); |
| 202 |
|
| 203 |
self.last_inc = self.mem.len(); |
| 204 |
let header = MthdHeader::new( |
| 205 |
MthdType::ZeroInc, |
| 206 |
class_to_subc(M::CLASS), |
| 207 |
M::ADDR, |
| 208 |
0, |
| 209 |
); |
| 210 |
self.mem.push(header.into_bits()); |
| 211 |
} |
| 212 |
|
| 213 |
pub fn push_mthd_1inc<M: Mthd>(&mut self, mthd: M) { |
| 214 |
self.last_inc = self.mem.len(); |
| 215 |
let header = MthdHeader::new( |
| 216 |
MthdType::OneInc, |
| 217 |
class_to_subc(M::CLASS), |
| 218 |
M::ADDR, |
| 219 |
0, |
| 220 |
); |
| 221 |
self.mem.push(header.into_bits()); |
| 222 |
self.mem.push(mthd.to_bits()); |
| 223 |
} |
| 224 |
|
| 225 |
pub fn push_array_method<M: ArrayMthd>(&mut self, i: usize, mthd: M) { |
| 226 |
self.push_mthd_bits( |
| 227 |
class_to_subc(M::CLASS), |
| 228 |
M::addr(i), |
| 229 |
mthd.to_bits(), |
| 230 |
); |
| 231 |
} |
| 232 |
|
| 233 |
pub fn push_array_mthd_0inc<M: ArrayMthd>(&mut self, i: usize, mthd: M) { |
| 234 |
assert!(mthd.to_bits() == 0); |
| 235 |
|
| 236 |
self.last_inc = self.mem.len(); |
| 237 |
let header = MthdHeader::new( |
| 238 |
MthdType::ZeroInc, |
| 239 |
class_to_subc(M::CLASS), |
| 240 |
M::addr(i), |
| 241 |
0, |
| 242 |
); |
| 243 |
self.mem.push(header.into_bits()); |
| 244 |
} |
| 245 |
|
| 246 |
pub fn push_array_mthd_1inc<M: ArrayMthd>(&mut self, i: usize, mthd: M) { |
| 247 |
self.last_inc = self.mem.len(); |
| 248 |
let header = MthdHeader::new( |
| 249 |
MthdType::OneInc, |
| 250 |
class_to_subc(M::CLASS), |
| 251 |
M::addr(i), |
| 252 |
1, |
| 253 |
); |
| 254 |
self.mem.push(header.into_bits()); |
| 255 |
self.mem.push(mthd.to_bits()); |
| 256 |
} |
| 257 |
|
| 258 |
/// Push an array of dwords into the push buffer |
| 259 |
pub fn push_inline_data(&mut self, data: &[u32]) { |
| 260 |
if let Some(last) = self.mem.get_mut(self.last_inc) { |
| 261 |
let last = MthdHeader::from_bits_mut(last); |
| 262 |
last.add_len(data.len().try_into().unwrap()); |
| 263 |
} else { |
| 264 |
panic!("Inline data must only be placed after a method header"); |
| 265 |
} |
| 266 |
self.mem.extend_from_slice(data); |
| 267 |
} |
| 268 |
} |
| 269 |
|
| 270 |
impl Default for Push { |
| 271 |
fn default() -> Self { |
| 272 |
Self::new() |
| 273 |
} |
| 274 |
} |
| 275 |
|
| 276 |
impl std::ops::Deref for Push { |
| 277 |
type Target = [u32]; |
| 278 |
|
| 279 |
fn deref(&self) -> &[u32] { |
| 280 |
&self.mem |
| 281 |
} |
| 282 |
} |