| 1 |
#ifndef NV_PUSH_H |
| 2 |
#define NV_PUSH_H |
| 3 |
|
| 4 |
#include "nvtypes.h" |
| 5 |
#include "util/macros.h" |
| 6 |
|
| 7 |
#include "drf.h" |
| 8 |
#include "cl906f.h" |
| 9 |
|
| 10 |
#include <assert.h> |
| 11 |
#include <stddef.h> |
| 12 |
#include <string.h> |
| 13 |
#include <stdio.h> |
| 14 |
|
| 15 |
struct nv_device_info; |
| 16 |
|
| 17 |
struct nv_push { |
| 18 |
uint32_t *start; |
| 19 |
uint32_t *end; |
| 20 |
uint32_t *limit; |
| 21 |
|
| 22 |
/* A pointer to the last method header */ |
| 23 |
uint32_t *last_hdr; |
| 24 |
|
| 25 |
/* The value in the last method header, used to avoid read-back */ |
| 26 |
uint32_t last_hdr_dw; |
| 27 |
|
| 28 |
#ifndef NDEBUG |
| 29 |
/* A mask of valid subchannels */ |
| 30 |
uint8_t subc_mask; |
| 31 |
#endif |
| 32 |
}; |
| 33 |
|
| 34 |
#define SUBC_MASK_ALL 0xff |
| 35 |
|
| 36 |
static inline void |
| 37 |
nv_push_init(struct nv_push *push, uint32_t *start, |
| 38 |
size_t dw_count, uint8_t subc_mask) |
| 39 |
{ |
| 40 |
push->start = start; |
| 41 |
push->end = start; |
| 42 |
push->limit = start + dw_count; |
| 43 |
push->last_hdr = NULL; |
| 44 |
push->last_hdr_dw = 0; |
| 45 |
|
| 46 |
#ifndef NDEBUG |
| 47 |
push->subc_mask = subc_mask; |
| 48 |
#endif |
| 49 |
} |
| 50 |
|
| 51 |
static inline size_t |
| 52 |
nv_push_dw_count(const struct nv_push *push) |
| 53 |
{ |
| 54 |
assert(push->start <= push->end); |
| 55 |
assert(push->end <= push->limit); |
| 56 |
return push->end - push->start; |
| 57 |
} |
| 58 |
|
| 59 |
#ifndef NDEBUG |
| 60 |
void nv_push_validate(struct nv_push *push); |
| 61 |
#else |
| 62 |
static inline void nv_push_validate(struct nv_push *push) { } |
| 63 |
#endif |
| 64 |
|
| 65 |
void vk_push_print(FILE *fp, const struct nv_push *push, |
| 66 |
const struct nv_device_info *devinfo) ATTRIBUTE_COLD; |
| 67 |
|
| 68 |
#define SUBC_NV9097 0 |
| 69 |
#define SUBC_NV9297 0 |
| 70 |
#define SUBC_NVA097 0 |
| 71 |
#define SUBC_NVB097 0 |
| 72 |
#define SUBC_NVB197 0 |
| 73 |
#define SUBC_NVC097 0 |
| 74 |
#define SUBC_NVC397 0 |
| 75 |
#define SUBC_NVC597 0 |
| 76 |
#define SUBC_NVC797 0 |
| 77 |
#define SUBC_NVC86F 0 |
| 78 |
#define SUBC_NVCB97 0 |
| 79 |
#define SUBC_NVCD97 0 |
| 80 |
|
| 81 |
#define SUBC_NV90C0 1 |
| 82 |
#define SUBC_NVA0C0 1 |
| 83 |
#define SUBC_NVB0C0 1 |
| 84 |
#define SUBC_NVB1C0 1 |
| 85 |
#define SUBC_NVC0C0 1 |
| 86 |
#define SUBC_NVC3C0 1 |
| 87 |
#define SUBC_NVC6C0 1 |
| 88 |
#define SUBC_NVC7C0 1 |
| 89 |
|
| 90 |
#define SUBC_NV9039 2 |
| 91 |
|
| 92 |
#define SUBC_NV902D 3 |
| 93 |
|
| 94 |
#define SUBC_NV90B5 4 |
| 95 |
#define SUBC_NVC1B5 4 |
| 96 |
|
| 97 |
/* video decode will get push on sub channel 4 */ |
| 98 |
#define SUBC_NVC5B0 4 |
| 99 |
|
| 100 |
static inline uint8_t |
| 101 |
NVC0_FIFO_SUBC_FROM_PKHDR(uint32_t hdr) |
| 102 |
{ |
| 103 |
return NVVAL_GET(hdr, NV906F, DMA, METHOD_SUBCHANNEL); |
| 104 |
} |
| 105 |
|
| 106 |
static inline uint32_t |
| 107 |
NVC0_FIFO_PKHDR_SQ(int subc, int mthd, unsigned size) |
| 108 |
{ |
| 109 |
return NVDEF(NV906F, DMA_INCR, OPCODE, VALUE) | |
| 110 |
NVVAL(NV906F, DMA_INCR, COUNT, size) | |
| 111 |
NVVAL(NV906F, DMA_INCR, SUBCHANNEL, subc) | |
| 112 |
NVVAL(NV906F, DMA_INCR, ADDRESS, mthd >> 2); |
| 113 |
} |
| 114 |
|
| 115 |
static inline void |
| 116 |
__push_verify(struct nv_push *push) |
| 117 |
{ |
| 118 |
if (push->last_hdr == NULL) |
| 119 |
return; |
| 120 |
|
| 121 |
/* check for immd */ |
| 122 |
if (NVDEF_TEST(push->last_hdr_dw, NV906F, DMA, SEC_OP, ==, IMMD_DATA_METHOD)) |
| 123 |
return; |
| 124 |
|
| 125 |
ASSERTED uint32_t last_count = |
| 126 |
NVVAL_GET(push->last_hdr_dw, NV906F, DMA, METHOD_COUNT); |
| 127 |
assert(last_count); |
| 128 |
} |
| 129 |
|
| 130 |
static inline void |
| 131 |
__push_hdr(struct nv_push *push, uint32_t hdr) |
| 132 |
{ |
| 133 |
__push_verify(push); |
| 134 |
|
| 135 |
ASSERTED const uint32_t subc = NVC0_FIFO_SUBC_FROM_PKHDR(hdr); |
| 136 |
assert(push->subc_mask & BITFIELD_BIT(subc)); |
| 137 |
|
| 138 |
*push->end = hdr; |
| 139 |
push->last_hdr_dw = hdr; |
| 140 |
push->last_hdr = push->end; |
| 141 |
push->end++; |
| 142 |
} |
| 143 |
|
| 144 |
static inline void |
| 145 |
__push_mthd_size(struct nv_push *push, int subc, uint32_t mthd, unsigned size) |
| 146 |
{ |
| 147 |
__push_hdr(push, NVC0_FIFO_PKHDR_SQ(subc, mthd, size)); |
| 148 |
} |
| 149 |
|
| 150 |
static inline void |
| 151 |
__push_mthd(struct nv_push *push, int subc, uint32_t mthd) |
| 152 |
{ |
| 153 |
__push_mthd_size(push, subc, mthd, 0); |
| 154 |
} |
| 155 |
|
| 156 |
#define P_MTHD(push, class, mthd) __push_mthd(push, SUBC_##class, class##_##mthd) |
| 157 |
|
| 158 |
static inline uint32_t |
| 159 |
NVC0_FIFO_PKHDR_IL(int subc, int mthd, uint16_t data) |
| 160 |
{ |
| 161 |
assert(!(data & ~DRF_MASK(NV906F_DMA_IMMD_DATA))); |
| 162 |
return NVDEF(NV906F, DMA_IMMD, OPCODE, VALUE) | |
| 163 |
NVVAL(NV906F, DMA_IMMD, DATA, data) | |
| 164 |
NVVAL(NV906F, DMA_IMMD, SUBCHANNEL, subc) | |
| 165 |
NVVAL(NV906F, DMA_IMMD, ADDRESS, mthd >> 2); |
| 166 |
} |
| 167 |
|
| 168 |
static inline void |
| 169 |
__push_immd(struct nv_push *push, int subc, uint32_t mthd, uint32_t val) |
| 170 |
{ |
| 171 |
__push_hdr(push, NVC0_FIFO_PKHDR_IL(subc, mthd, val)); |
| 172 |
} |
| 173 |
|
| 174 |
/** |
| 175 |
* P_IMMD() pushes a command and its arguments. Note that this always |
| 176 |
* takes two words in unoptimized builds but may sometimes take one word |
| 177 |
* with optimizations on. |
| 178 |
*/ |
| 179 |
#define P_IMMD(push, class, mthd, args...) do { \ |
| 180 |
uint32_t __val; \ |
| 181 |
VA_##class##_##mthd(__val, args); \ |
| 182 |
if (__builtin_constant_p(__val & ~0x1fff) && !(__val & ~0x1fff)) { \ |
| 183 |
__push_immd(push, SUBC_##class, class##_##mthd, __val); \ |
| 184 |
} else { \ |
| 185 |
__push_mthd_size(push, SUBC_##class, class##_##mthd, 0); \ |
| 186 |
nv_push_val(push, class##_##mthd, __val); \ |
| 187 |
} \ |
| 188 |
} while(0) |
| 189 |
|
| 190 |
/** |
| 191 |
* P_IMMD_WORD is the same as P_IMMD, except that it always takes exactly |
| 192 |
* one word of space (and asserts if two words would be needed) |
| 193 |
*/ |
| 194 |
#define P_IMMD_WORD(push, class, mthd, args...) do { \ |
| 195 |
uint32_t __val; \ |
| 196 |
VA_##class##_##mthd(__val, args); \ |
| 197 |
assert(!(__val & ~0x1fff)); \ |
| 198 |
__push_immd(push, SUBC_##class, class##_##mthd, __val); \ |
| 199 |
} while(0) |
| 200 |
|
| 201 |
static inline uint32_t |
| 202 |
NVC0_FIFO_PKHDR_1I(int subc, int mthd, unsigned size) |
| 203 |
{ |
| 204 |
return NVDEF(NV906F, DMA_ONEINCR, OPCODE, VALUE) | |
| 205 |
NVVAL(NV906F, DMA_ONEINCR, COUNT, size) | |
| 206 |
NVVAL(NV906F, DMA_ONEINCR, SUBCHANNEL, subc) | |
| 207 |
NVVAL(NV906F, DMA_ONEINCR, ADDRESS, mthd >> 2); |
| 208 |
} |
| 209 |
|
| 210 |
static inline void |
| 211 |
__push_1inc(struct nv_push *push, int subc, uint32_t mthd) |
| 212 |
{ |
| 213 |
__push_hdr(push, NVC0_FIFO_PKHDR_1I(subc, mthd, 0)); |
| 214 |
} |
| 215 |
|
| 216 |
#define P_1INC(push, class, mthd) __push_1inc(push, SUBC_##class, class##_##mthd) |
| 217 |
|
| 218 |
static inline uint32_t |
| 219 |
NVC0_FIFO_PKHDR_0I(int subc, int mthd, unsigned size) |
| 220 |
{ |
| 221 |
return NVDEF(NV906F, DMA_NONINCR, OPCODE, VALUE) | |
| 222 |
NVVAL(NV906F, DMA_NONINCR, COUNT, size) | |
| 223 |
NVVAL(NV906F, DMA_NONINCR, SUBCHANNEL, subc) | |
| 224 |
NVVAL(NV906F, DMA_NONINCR, ADDRESS, mthd >> 2); |
| 225 |
} |
| 226 |
|
| 227 |
static inline void |
| 228 |
__push_0inc(struct nv_push *push, int subc, uint32_t mthd) |
| 229 |
{ |
| 230 |
__push_hdr(push, NVC0_FIFO_PKHDR_0I(subc, mthd, 0)); |
| 231 |
} |
| 232 |
|
| 233 |
#define P_0INC(push, class, mthd) __push_0inc(push, SUBC_##class, class##_##mthd) |
| 234 |
|
| 235 |
#define NV_PUSH_MAX_COUNT DRF_MASK(NV906F_DMA_METHOD_COUNT) |
| 236 |
|
| 237 |
static inline bool |
| 238 |
nv_push_update_count(struct nv_push *push, uint16_t count) |
| 239 |
{ |
| 240 |
assert(push->last_hdr != NULL); |
| 241 |
|
| 242 |
assert(count <= NV_PUSH_MAX_COUNT); |
| 243 |
if (count > NV_PUSH_MAX_COUNT) |
| 244 |
return false; |
| 245 |
|
| 246 |
uint32_t hdr_dw = push->last_hdr_dw; |
| 247 |
|
| 248 |
uint32_t old_count = NVVAL_GET(hdr_dw, NV906F, DMA, METHOD_COUNT); |
| 249 |
uint32_t new_count = (count + old_count) & NV_PUSH_MAX_COUNT; |
| 250 |
bool overflow = new_count < count; |
| 251 |
/* if we would overflow, don't change anything and just let it be */ |
| 252 |
assert(!overflow); |
| 253 |
if (overflow) |
| 254 |
return false; |
| 255 |
|
| 256 |
hdr_dw = NVVAL_SET(hdr_dw, NV906F, DMA, METHOD_COUNT, new_count); |
| 257 |
push->last_hdr_dw = hdr_dw; |
| 258 |
*push->last_hdr = hdr_dw; |
| 259 |
return true; |
| 260 |
} |
| 261 |
|
| 262 |
static inline void |
| 263 |
P_INLINE_DATA(struct nv_push *push, uint32_t value) |
| 264 |
{ |
| 265 |
assert(push->end < push->limit); |
| 266 |
if (nv_push_update_count(push, 1)) { |
| 267 |
/* push new value */ |
| 268 |
*push->end = value; |
| 269 |
push->end++; |
| 270 |
} |
| 271 |
} |
| 272 |
|
| 273 |
static inline void |
| 274 |
P_INLINE_FLOAT(struct nv_push *push, float value) |
| 275 |
{ |
| 276 |
assert(push->end < push->limit); |
| 277 |
if (nv_push_update_count(push, 1)) { |
| 278 |
/* push new value */ |
| 279 |
*(float *)push->end = value; |
| 280 |
push->end++; |
| 281 |
} |
| 282 |
} |
| 283 |
|
| 284 |
static inline void |
| 285 |
P_INLINE_ARRAY(struct nv_push *push, const uint32_t *data, int num_dw) |
| 286 |
{ |
| 287 |
assert(push->end + num_dw <= push->limit); |
| 288 |
if (nv_push_update_count(push, num_dw)) { |
| 289 |
/* push new value */ |
| 290 |
memcpy(push->end, data, num_dw * 4); |
| 291 |
push->end += num_dw; |
| 292 |
} |
| 293 |
} |
| 294 |
|
| 295 |
/* internally used by generated inlines. */ |
| 296 |
static inline void |
| 297 |
nv_push_val(struct nv_push *push, uint32_t idx, uint32_t val) |
| 298 |
{ |
| 299 |
ASSERTED uint32_t last_hdr_dw = push->last_hdr_dw; |
| 300 |
ASSERTED uint32_t type = NVVAL_GET(last_hdr_dw, NV906F, DMA, SEC_OP); |
| 301 |
ASSERTED bool is_0inc = type == NV906F_DMA_SEC_OP_NON_INC_METHOD; |
| 302 |
ASSERTED bool is_1inc = type == NV906F_DMA_SEC_OP_ONE_INC; |
| 303 |
ASSERTED bool is_immd = type == NV906F_DMA_SEC_OP_IMMD_DATA_METHOD; |
| 304 |
ASSERTED uint16_t last_method = |
| 305 |
NVVAL_GET(last_hdr_dw, NV906F, DMA, METHOD_ADDRESS) << 2; |
| 306 |
|
| 307 |
uint16_t distance = push->end - push->last_hdr - 1; |
| 308 |
if (is_0inc) |
| 309 |
distance = 0; |
| 310 |
else if (is_1inc) |
| 311 |
distance = MIN2(1, distance); |
| 312 |
last_method += distance * 4; |
| 313 |
|
| 314 |
/* can't have empty headers ever */ |
| 315 |
assert(last_hdr_dw); |
| 316 |
assert(!is_immd); |
| 317 |
assert(last_method == idx); |
| 318 |
assert(push->end < push->limit); |
| 319 |
|
| 320 |
P_INLINE_DATA(push, val); |
| 321 |
} |
| 322 |
|
| 323 |
static inline void |
| 324 |
nv_push_raw(struct nv_push *push, uint32_t *raw_dw, uint32_t dw_count) |
| 325 |
{ |
| 326 |
assert(push->end + dw_count <= push->limit); |
| 327 |
memcpy(push->end, raw_dw, dw_count * 4); |
| 328 |
push->end += dw_count; |
| 329 |
push->last_hdr = NULL; |
| 330 |
} |
| 331 |
|
| 332 |
#endif /* NV_PUSH_H */ |