| 1 |
// SPDX-License-Identifier: GPL-2.0 |
| 2 |
|
| 3 |
/* Copyright (c) 2012-2018, The Linux Foundation. All rights reserved. |
| 4 |
* Copyright (C) 2019-2024 Linaro Ltd. |
| 5 |
*/ |
| 6 |
|
| 7 |
#include <linux/bitfield.h> |
| 8 |
#include <linux/bits.h> |
| 9 |
#include <linux/dma-direction.h> |
| 10 |
#include <linux/refcount.h> |
| 11 |
#include <linux/scatterlist.h> |
| 12 |
#include <linux/types.h> |
| 13 |
|
| 14 |
#include "gsi.h" |
| 15 |
#include "gsi_private.h" |
| 16 |
#include "gsi_trans.h" |
| 17 |
#include "ipa_cmd.h" |
| 18 |
#include "ipa_data.h" |
| 19 |
#include "ipa_gsi.h" |
| 20 |
|
| 21 |
/** |
| 22 |
* DOC: GSI Transactions |
| 23 |
* |
| 24 |
* A GSI transaction abstracts the behavior of a GSI channel by representing |
| 25 |
* everything about a related group of IPA operations in a single structure. |
| 26 |
* (A "operation" in this sense is either a data transfer or an IPA immediate |
| 27 |
* command.) Most details of interaction with the GSI hardware are managed |
| 28 |
* by the GSI transaction core, allowing users to simply describe operations |
| 29 |
* to be performed. When a transaction has completed a callback function |
| 30 |
* (dependent on the type of endpoint associated with the channel) allows |
| 31 |
* cleanup of resources associated with the transaction. |
| 32 |
* |
| 33 |
* To perform an operation (or set of them), a user of the GSI transaction |
| 34 |
* interface allocates a transaction, indicating the number of TREs required |
| 35 |
* (one per operation). If sufficient TREs are available, they are reserved |
| 36 |
* for use in the transaction and the allocation succeeds. This way |
| 37 |
* exhaustion of the available TREs in a channel ring is detected as early |
| 38 |
* as possible. Any other resources that might be needed to complete a |
| 39 |
* transaction are also allocated when the transaction is allocated. |
| 40 |
* |
| 41 |
* Operations performed as part of a transaction are represented in an array |
| 42 |
* of Linux scatterlist structures, allocated with the transaction. These |
| 43 |
* scatterlist structures are initialized by "adding" operations to the |
| 44 |
* transaction. If a buffer in an operation must be mapped for DMA, this is |
| 45 |
* done at the time it is added to the transaction. It is possible for a |
| 46 |
* mapping error to occur when an operation is added. In this case the |
| 47 |
* transaction should simply be freed; this correctly releases resources |
| 48 |
* associated with the transaction. |
| 49 |
* |
| 50 |
* Once all operations have been successfully added to a transaction, the |
| 51 |
* transaction is committed. Committing transfers ownership of the entire |
| 52 |
* transaction to the GSI transaction core. The GSI transaction code |
| 53 |
* formats the content of the scatterlist array into the channel ring |
| 54 |
* buffer and informs the hardware that new TREs are available to process. |
| 55 |
* |
| 56 |
* The last TRE in each transaction is marked to interrupt the AP when the |
| 57 |
* GSI hardware has completed it. Because transfers described by TREs are |
| 58 |
* performed strictly in order, signaling the completion of just the last |
| 59 |
* TRE in the transaction is sufficient to indicate the full transaction |
| 60 |
* is complete. |
| 61 |
* |
| 62 |
* When a transaction is complete, ipa_gsi_trans_complete() is called by the |
| 63 |
* GSI code into the IPA layer, allowing it to perform any final cleanup |
| 64 |
* required before the transaction is freed. |
| 65 |
*/ |
| 66 |
|
| 67 |
/* Hardware values representing a transfer element type */ |
| 68 |
enum gsi_tre_type { |
| 69 |
GSI_RE_XFER = 0x2, |
| 70 |
GSI_RE_IMMD_CMD = 0x3, |
| 71 |
}; |
| 72 |
|
| 73 |
/* An entry in a channel ring */ |
| 74 |
struct gsi_tre { |
| 75 |
__le64 addr; /* DMA address */ |
| 76 |
__le16 len_opcode; /* length in bytes or enum IPA_CMD_* */ |
| 77 |
__le16 reserved; |
| 78 |
__le32 flags; /* TRE_FLAGS_* */ |
| 79 |
}; |
| 80 |
|
| 81 |
/* gsi_tre->flags mask values (in CPU byte order) */ |
| 82 |
#define TRE_FLAGS_CHAIN_FMASK GENMASK(0, 0) |
| 83 |
#define TRE_FLAGS_IEOT_FMASK GENMASK(9, 9) |
| 84 |
#define TRE_FLAGS_BEI_FMASK GENMASK(10, 10) |
| 85 |
#define TRE_FLAGS_TYPE_FMASK GENMASK(23, 16) |
| 86 |
|
| 87 |
int gsi_trans_pool_init(struct gsi_trans_pool *pool, size_t size, u32 count, |
| 88 |
u32 max_alloc) |
| 89 |
{ |
| 90 |
size_t alloc_size; |
| 91 |
void *virt; |
| 92 |
|
| 93 |
if (!size) |
| 94 |
return -EINVAL; |
| 95 |
if (count < max_alloc) |
| 96 |
return -EINVAL; |
| 97 |
if (!max_alloc) |
| 98 |
return -EINVAL; |
| 99 |
|
| 100 |
/* By allocating a few extra entries in our pool (one less |
| 101 |
* than the maximum number that will be requested in a |
| 102 |
* single allocation), we can always satisfy requests without |
| 103 |
* ever worrying about straddling the end of the pool array. |
| 104 |
* If there aren't enough entries starting at the free index, |
| 105 |
* we just allocate free entries from the beginning of the pool. |
| 106 |
*/ |
| 107 |
alloc_size = size_mul(count + max_alloc - 1, size); |
| 108 |
alloc_size = kmalloc_size_roundup(alloc_size); |
| 109 |
virt = kzalloc(alloc_size, GFP_KERNEL); |
| 110 |
if (!virt) |
| 111 |
return -ENOMEM; |
| 112 |
|
| 113 |
pool->base = virt; |
| 114 |
/* If the allocator gave us any extra memory, use it */ |
| 115 |
pool->count = alloc_size / size; |
| 116 |
pool->free = 0; |
| 117 |
pool->max_alloc = max_alloc; |
| 118 |
pool->size = size; |
| 119 |
pool->addr = 0; /* Only used for DMA pools */ |
| 120 |
|
| 121 |
return 0; |
| 122 |
} |
| 123 |
|
| 124 |
void gsi_trans_pool_exit(struct gsi_trans_pool *pool) |
| 125 |
{ |
| 126 |
kfree(pool->base); |
| 127 |
memset(pool, 0, sizeof(*pool)); |
| 128 |
} |
| 129 |
|
| 130 |
/* Home-grown DMA pool. This way we can preallocate the pool, and guarantee |
| 131 |
* allocations will succeed. The immediate commands in a transaction can |
| 132 |
* require up to max_alloc elements from the pool. But we only allow |
| 133 |
* allocation of a single element from a DMA pool at a time. |
| 134 |
*/ |
| 135 |
int gsi_trans_pool_init_dma(struct device *dev, struct gsi_trans_pool *pool, |
| 136 |
size_t size, u32 count, u32 max_alloc) |
| 137 |
{ |
| 138 |
size_t total_size; |
| 139 |
dma_addr_t addr; |
| 140 |
void *virt; |
| 141 |
|
| 142 |
if (!size) |
| 143 |
return -EINVAL; |
| 144 |
if (count < max_alloc) |
| 145 |
return -EINVAL; |
| 146 |
if (!max_alloc) |
| 147 |
return -EINVAL; |
| 148 |
|
| 149 |
/* Don't let allocations cross a power-of-two boundary */ |
| 150 |
size = __roundup_pow_of_two(size); |
| 151 |
total_size = (count + max_alloc - 1) * size; |
| 152 |
|
| 153 |
/* The allocator will give us a power-of-2 number of pages |
| 154 |
* sufficient to satisfy our request. Round up our requested |
| 155 |
* size to avoid any unused space in the allocation. This way |
| 156 |
* gsi_trans_pool_exit_dma() can assume the total allocated |
| 157 |
* size is exactly (count * size). |
| 158 |
*/ |
| 159 |
total_size = PAGE_SIZE << get_order(total_size); |
| 160 |
|
| 161 |
virt = dma_alloc_coherent(dev, total_size, &addr, GFP_KERNEL); |
| 162 |
if (!virt) |
| 163 |
return -ENOMEM; |
| 164 |
|
| 165 |
pool->base = virt; |
| 166 |
pool->count = total_size / size; |
| 167 |
pool->free = 0; |
| 168 |
pool->size = size; |
| 169 |
pool->max_alloc = max_alloc; |
| 170 |
pool->addr = addr; |
| 171 |
|
| 172 |
return 0; |
| 173 |
} |
| 174 |
|
| 175 |
void gsi_trans_pool_exit_dma(struct device *dev, struct gsi_trans_pool *pool) |
| 176 |
{ |
| 177 |
size_t total_size = pool->count * pool->size; |
| 178 |
|
| 179 |
dma_free_coherent(dev, total_size, pool->base, pool->addr); |
| 180 |
memset(pool, 0, sizeof(*pool)); |
| 181 |
} |
| 182 |
|
| 183 |
/* Return the byte offset of the next free entry in the pool */ |
| 184 |
static u32 gsi_trans_pool_alloc_common(struct gsi_trans_pool *pool, u32 count) |
| 185 |
{ |
| 186 |
u32 offset; |
| 187 |
|
| 188 |
WARN_ON(!count); |
| 189 |
WARN_ON(count > pool->max_alloc); |
| 190 |
|
| 191 |
/* Allocate from beginning if wrap would occur */ |
| 192 |
if (count > pool->count - pool->free) |
| 193 |
pool->free = 0; |
| 194 |
|
| 195 |
offset = pool->free * pool->size; |
| 196 |
pool->free += count; |
| 197 |
memset(pool->base + offset, 0, count * pool->size); |
| 198 |
|
| 199 |
return offset; |
| 200 |
} |
| 201 |
|
| 202 |
/* Allocate a contiguous block of zeroed entries from a pool */ |
| 203 |
void *gsi_trans_pool_alloc(struct gsi_trans_pool *pool, u32 count) |
| 204 |
{ |
| 205 |
return pool->base + gsi_trans_pool_alloc_common(pool, count); |
| 206 |
} |
| 207 |
|
| 208 |
/* Allocate a single zeroed entry from a DMA pool */ |
| 209 |
void *gsi_trans_pool_alloc_dma(struct gsi_trans_pool *pool, dma_addr_t *addr) |
| 210 |
{ |
| 211 |
u32 offset = gsi_trans_pool_alloc_common(pool, 1); |
| 212 |
|
| 213 |
*addr = pool->addr + offset; |
| 214 |
|
| 215 |
return pool->base + offset; |
| 216 |
} |
| 217 |
|
| 218 |
/* Map a TRE ring entry index to the transaction it is associated with */ |
| 219 |
static void gsi_trans_map(struct gsi_trans *trans, u32 index) |
| 220 |
{ |
| 221 |
struct gsi_channel *channel = &trans->gsi->channel[trans->channel_id]; |
| 222 |
|
| 223 |
/* The completion event will indicate the last TRE used */ |
| 224 |
index += trans->used_count - 1; |
| 225 |
|
| 226 |
/* Note: index *must* be used modulo the ring count here */ |
| 227 |
channel->trans_info.map[index % channel->tre_ring.count] = trans; |
| 228 |
} |
| 229 |
|
| 230 |
/* Return the transaction mapped to a given ring entry */ |
| 231 |
struct gsi_trans * |
| 232 |
gsi_channel_trans_mapped(struct gsi_channel *channel, u32 index) |
| 233 |
{ |
| 234 |
/* Note: index *must* be used modulo the ring count here */ |
| 235 |
return channel->trans_info.map[index % channel->tre_ring.count]; |
| 236 |
} |
| 237 |
|
| 238 |
/* Return the oldest completed transaction for a channel (or null) */ |
| 239 |
struct gsi_trans *gsi_channel_trans_complete(struct gsi_channel *channel) |
| 240 |
{ |
| 241 |
struct gsi_trans_info *trans_info = &channel->trans_info; |
| 242 |
u16 trans_id = trans_info->completed_id; |
| 243 |
|
| 244 |
if (trans_id == trans_info->pending_id) { |
| 245 |
gsi_channel_update(channel); |
| 246 |
if (trans_id == trans_info->pending_id) |
| 247 |
return NULL; |
| 248 |
} |
| 249 |
|
| 250 |
return &trans_info->trans[trans_id %= channel->tre_count]; |
| 251 |
} |
| 252 |
|
| 253 |
/* Move a transaction from allocated to committed state */ |
| 254 |
static void gsi_trans_move_committed(struct gsi_trans *trans) |
| 255 |
{ |
| 256 |
struct gsi_channel *channel = &trans->gsi->channel[trans->channel_id]; |
| 257 |
struct gsi_trans_info *trans_info = &channel->trans_info; |
| 258 |
|
| 259 |
/* This allocated transaction is now committed */ |
| 260 |
trans_info->allocated_id++; |
| 261 |
} |
| 262 |
|
| 263 |
/* Move committed transactions to pending state */ |
| 264 |
static void gsi_trans_move_pending(struct gsi_trans *trans) |
| 265 |
{ |
| 266 |
struct gsi_channel *channel = &trans->gsi->channel[trans->channel_id]; |
| 267 |
struct gsi_trans_info *trans_info = &channel->trans_info; |
| 268 |
u16 trans_index = trans - &trans_info->trans[0]; |
| 269 |
u16 delta; |
| 270 |
|
| 271 |
/* These committed transactions are now pending */ |
| 272 |
delta = trans_index - trans_info->committed_id + 1; |
| 273 |
trans_info->committed_id += delta % channel->tre_count; |
| 274 |
} |
| 275 |
|
| 276 |
/* Move pending transactions to completed state */ |
| 277 |
void gsi_trans_move_complete(struct gsi_trans *trans) |
| 278 |
{ |
| 279 |
struct gsi_channel *channel = &trans->gsi->channel[trans->channel_id]; |
| 280 |
struct gsi_trans_info *trans_info = &channel->trans_info; |
| 281 |
u16 trans_index = trans - trans_info->trans; |
| 282 |
u16 delta; |
| 283 |
|
| 284 |
/* These pending transactions are now completed */ |
| 285 |
delta = trans_index - trans_info->pending_id + 1; |
| 286 |
delta %= channel->tre_count; |
| 287 |
trans_info->pending_id += delta; |
| 288 |
} |
| 289 |
|
| 290 |
/* Move a transaction from completed to polled state */ |
| 291 |
void gsi_trans_move_polled(struct gsi_trans *trans) |
| 292 |
{ |
| 293 |
struct gsi_channel *channel = &trans->gsi->channel[trans->channel_id]; |
| 294 |
struct gsi_trans_info *trans_info = &channel->trans_info; |
| 295 |
|
| 296 |
/* This completed transaction is now polled */ |
| 297 |
trans_info->completed_id++; |
| 298 |
} |
| 299 |
|
| 300 |
/* Reserve some number of TREs on a channel. Returns true if successful */ |
| 301 |
static bool |
| 302 |
gsi_trans_tre_reserve(struct gsi_trans_info *trans_info, u32 tre_count) |
| 303 |
{ |
| 304 |
int avail = atomic_read(&trans_info->tre_avail); |
| 305 |
int new; |
| 306 |
|
| 307 |
do { |
| 308 |
new = avail - (int)tre_count; |
| 309 |
if (unlikely(new < 0)) |
| 310 |
return false; |
| 311 |
} while (!atomic_try_cmpxchg(&trans_info->tre_avail, &avail, new)); |
| 312 |
|
| 313 |
return true; |
| 314 |
} |
| 315 |
|
| 316 |
/* Release previously-reserved TRE entries to a channel */ |
| 317 |
static void |
| 318 |
gsi_trans_tre_release(struct gsi_trans_info *trans_info, u32 tre_count) |
| 319 |
{ |
| 320 |
atomic_add(tre_count, &trans_info->tre_avail); |
| 321 |
} |
| 322 |
|
| 323 |
/* Return true if no transactions are allocated, false otherwise */ |
| 324 |
bool gsi_channel_trans_idle(struct gsi *gsi, u32 channel_id) |
| 325 |
{ |
| 326 |
u32 tre_max = gsi_channel_tre_max(gsi, channel_id); |
| 327 |
struct gsi_trans_info *trans_info; |
| 328 |
|
| 329 |
trans_info = &gsi->channel[channel_id].trans_info; |
| 330 |
|
| 331 |
return atomic_read(&trans_info->tre_avail) == tre_max; |
| 332 |
} |
| 333 |
|
| 334 |
/* Allocate a GSI transaction on a channel */ |
| 335 |
struct gsi_trans *gsi_channel_trans_alloc(struct gsi *gsi, u32 channel_id, |
| 336 |
u32 tre_count, |
| 337 |
enum dma_data_direction direction) |
| 338 |
{ |
| 339 |
struct gsi_channel *channel = &gsi->channel[channel_id]; |
| 340 |
struct gsi_trans_info *trans_info; |
| 341 |
struct gsi_trans *trans; |
| 342 |
u16 trans_index; |
| 343 |
|
| 344 |
if (WARN_ON(tre_count > channel->trans_tre_max)) |
| 345 |
return NULL; |
| 346 |
|
| 347 |
trans_info = &channel->trans_info; |
| 348 |
|
| 349 |
/* If we can't reserve the TREs for the transaction, we're done */ |
| 350 |
if (!gsi_trans_tre_reserve(trans_info, tre_count)) |
| 351 |
return NULL; |
| 352 |
|
| 353 |
trans_index = trans_info->free_id % channel->tre_count; |
| 354 |
trans = &trans_info->trans[trans_index]; |
| 355 |
memset(trans, 0, sizeof(*trans)); |
| 356 |
|
| 357 |
/* Initialize non-zero fields in the transaction */ |
| 358 |
trans->gsi = gsi; |
| 359 |
trans->channel_id = channel_id; |
| 360 |
trans->rsvd_count = tre_count; |
| 361 |
init_completion(&trans->completion); |
| 362 |
|
| 363 |
/* Allocate the scatterlist */ |
| 364 |
trans->sgl = gsi_trans_pool_alloc(&trans_info->sg_pool, tre_count); |
| 365 |
sg_init_marker(trans->sgl, tre_count); |
| 366 |
|
| 367 |
trans->direction = direction; |
| 368 |
refcount_set(&trans->refcount, 1); |
| 369 |
|
| 370 |
/* This free transaction is now allocated */ |
| 371 |
trans_info->free_id++; |
| 372 |
|
| 373 |
return trans; |
| 374 |
} |
| 375 |
|
| 376 |
/* Free a previously-allocated transaction */ |
| 377 |
void gsi_trans_free(struct gsi_trans *trans) |
| 378 |
{ |
| 379 |
struct gsi_trans_info *trans_info; |
| 380 |
|
| 381 |
if (!refcount_dec_and_test(&trans->refcount)) |
| 382 |
return; |
| 383 |
|
| 384 |
/* Unused transactions are allocated but never committed, pending, |
| 385 |
* completed, or polled. |
| 386 |
*/ |
| 387 |
trans_info = &trans->gsi->channel[trans->channel_id].trans_info; |
| 388 |
if (!trans->used_count) { |
| 389 |
trans_info->allocated_id++; |
| 390 |
trans_info->committed_id++; |
| 391 |
trans_info->pending_id++; |
| 392 |
trans_info->completed_id++; |
| 393 |
} else { |
| 394 |
ipa_gsi_trans_release(trans); |
| 395 |
} |
| 396 |
|
| 397 |
/* This transaction is now free */ |
| 398 |
trans_info->polled_id++; |
| 399 |
|
| 400 |
/* Releasing the reserved TREs implicitly frees the sgl[] and |
| 401 |
* (if present) info[] arrays, plus the transaction itself. |
| 402 |
*/ |
| 403 |
gsi_trans_tre_release(trans_info, trans->rsvd_count); |
| 404 |
} |
| 405 |
|
| 406 |
/* Add an immediate command to a transaction */ |
| 407 |
void gsi_trans_cmd_add(struct gsi_trans *trans, void *buf, u32 size, |
| 408 |
dma_addr_t addr, enum ipa_cmd_opcode opcode) |
| 409 |
{ |
| 410 |
u32 which = trans->used_count++; |
| 411 |
struct scatterlist *sg; |
| 412 |
|
| 413 |
WARN_ON(which >= trans->rsvd_count); |
| 414 |
|
| 415 |
/* Commands are quite different from data transfer requests. |
| 416 |
* Their payloads come from a pool whose memory is allocated |
| 417 |
* using dma_alloc_coherent(). We therefore do *not* map them |
| 418 |
* for DMA (unlike what we do for pages and skbs). |
| 419 |
* |
| 420 |
* When a transaction completes, the SGL is normally unmapped. |
| 421 |
* A command transaction has direction DMA_NONE, which tells |
| 422 |
* gsi_trans_complete() to skip the unmapping step. |
| 423 |
* |
| 424 |
* The only things we use directly in a command scatter/gather |
| 425 |
* entry are the DMA address and length. We still need the SG |
| 426 |
* table flags to be maintained though, so assign a NULL page |
| 427 |
* pointer for that purpose. |
| 428 |
*/ |
| 429 |
sg = &trans->sgl[which]; |
| 430 |
sg_assign_page(sg, NULL); |
| 431 |
sg_dma_address(sg) = addr; |
| 432 |
sg_dma_len(sg) = size; |
| 433 |
|
| 434 |
trans->cmd_opcode[which] = opcode; |
| 435 |
} |
| 436 |
|
| 437 |
/* Add a page transfer to a transaction. It will fill the only TRE. */ |
| 438 |
int gsi_trans_page_add(struct gsi_trans *trans, struct page *page, u32 size, |
| 439 |
u32 offset) |
| 440 |
{ |
| 441 |
struct scatterlist *sg = &trans->sgl[0]; |
| 442 |
int ret; |
| 443 |
|
| 444 |
if (WARN_ON(trans->rsvd_count != 1)) |
| 445 |
return -EINVAL; |
| 446 |
if (WARN_ON(trans->used_count)) |
| 447 |
return -EINVAL; |
| 448 |
|
| 449 |
sg_set_page(sg, page, size, offset); |
| 450 |
ret = dma_map_sg(trans->gsi->dev, sg, 1, trans->direction); |
| 451 |
if (!ret) |
| 452 |
return -ENOMEM; |
| 453 |
|
| 454 |
trans->used_count++; /* Transaction now owns the (DMA mapped) page */ |
| 455 |
|
| 456 |
return 0; |
| 457 |
} |
| 458 |
|
| 459 |
/* Add an SKB transfer to a transaction. No other TREs will be used. */ |
| 460 |
int gsi_trans_skb_add(struct gsi_trans *trans, struct sk_buff *skb) |
| 461 |
{ |
| 462 |
struct scatterlist *sg = &trans->sgl[0]; |
| 463 |
u32 used_count; |
| 464 |
int ret; |
| 465 |
|
| 466 |
if (WARN_ON(trans->rsvd_count != 1)) |
| 467 |
return -EINVAL; |
| 468 |
if (WARN_ON(trans->used_count)) |
| 469 |
return -EINVAL; |
| 470 |
|
| 471 |
/* skb->len will not be 0 (checked early) */ |
| 472 |
ret = skb_to_sgvec(skb, sg, 0, skb->len); |
| 473 |
if (ret < 0) |
| 474 |
return ret; |
| 475 |
used_count = ret; |
| 476 |
|
| 477 |
ret = dma_map_sg(trans->gsi->dev, sg, used_count, trans->direction); |
| 478 |
if (!ret) |
| 479 |
return -ENOMEM; |
| 480 |
|
| 481 |
/* Transaction now owns the (DMA mapped) skb */ |
| 482 |
trans->used_count += used_count; |
| 483 |
|
| 484 |
return 0; |
| 485 |
} |
| 486 |
|
| 487 |
/* Compute the length/opcode value to use for a TRE */ |
| 488 |
static __le16 gsi_tre_len_opcode(enum ipa_cmd_opcode opcode, u32 len) |
| 489 |
{ |
| 490 |
return opcode == IPA_CMD_NONE ? cpu_to_le16((u16)len) |
| 491 |
: cpu_to_le16((u16)opcode); |
| 492 |
} |
| 493 |
|
| 494 |
/* Compute the flags value to use for a given TRE */ |
| 495 |
static __le32 gsi_tre_flags(bool last_tre, bool bei, enum ipa_cmd_opcode opcode) |
| 496 |
{ |
| 497 |
enum gsi_tre_type tre_type; |
| 498 |
u32 tre_flags; |
| 499 |
|
| 500 |
tre_type = opcode == IPA_CMD_NONE ? GSI_RE_XFER : GSI_RE_IMMD_CMD; |
| 501 |
tre_flags = u32_encode_bits(tre_type, TRE_FLAGS_TYPE_FMASK); |
| 502 |
|
| 503 |
/* Last TRE contains interrupt flags */ |
| 504 |
if (last_tre) { |
| 505 |
/* All transactions end in a transfer completion interrupt */ |
| 506 |
tre_flags |= TRE_FLAGS_IEOT_FMASK; |
| 507 |
/* Don't interrupt when outbound commands are acknowledged */ |
| 508 |
if (bei) |
| 509 |
tre_flags |= TRE_FLAGS_BEI_FMASK; |
| 510 |
} else { /* All others indicate there's more to come */ |
| 511 |
tre_flags |= TRE_FLAGS_CHAIN_FMASK; |
| 512 |
} |
| 513 |
|
| 514 |
return cpu_to_le32(tre_flags); |
| 515 |
} |
| 516 |
|
| 517 |
static void gsi_trans_tre_fill(struct gsi_tre *dest_tre, dma_addr_t addr, |
| 518 |
u32 len, bool last_tre, bool bei, |
| 519 |
enum ipa_cmd_opcode opcode) |
| 520 |
{ |
| 521 |
struct gsi_tre tre; |
| 522 |
|
| 523 |
tre.addr = cpu_to_le64(addr); |
| 524 |
tre.len_opcode = gsi_tre_len_opcode(opcode, len); |
| 525 |
tre.reserved = 0; |
| 526 |
tre.flags = gsi_tre_flags(last_tre, bei, opcode); |
| 527 |
|
| 528 |
/* ARM64 can write 16 bytes as a unit with a single instruction. |
| 529 |
* Doing the assignment this way is an attempt to make that happen. |
| 530 |
*/ |
| 531 |
*dest_tre = tre; |
| 532 |
} |
| 533 |
|
| 534 |
/** |
| 535 |
* __gsi_trans_commit() - Common GSI transaction commit code |
| 536 |
* @trans: Transaction to commit |
| 537 |
* @ring_db: Whether to tell the hardware about these queued transfers |
| 538 |
* |
| 539 |
* Formats channel ring TRE entries based on the content of the scatterlist. |
| 540 |
* Maps a transaction pointer to the last ring entry used for the transaction, |
| 541 |
* so it can be recovered when it completes. Moves the transaction to |
| 542 |
* pending state. Finally, updates the channel ring pointer and optionally |
| 543 |
* rings the doorbell. |
| 544 |
*/ |
| 545 |
static void __gsi_trans_commit(struct gsi_trans *trans, bool ring_db) |
| 546 |
{ |
| 547 |
struct gsi_channel *channel = &trans->gsi->channel[trans->channel_id]; |
| 548 |
struct gsi_ring *tre_ring = &channel->tre_ring; |
| 549 |
enum ipa_cmd_opcode opcode = IPA_CMD_NONE; |
| 550 |
bool bei = channel->toward_ipa; |
| 551 |
struct gsi_tre *dest_tre; |
| 552 |
struct scatterlist *sg; |
| 553 |
u32 byte_count = 0; |
| 554 |
u8 *cmd_opcode; |
| 555 |
u32 avail; |
| 556 |
u32 i; |
| 557 |
|
| 558 |
WARN_ON(!trans->used_count); |
| 559 |
|
| 560 |
/* Consume the entries. If we cross the end of the ring while |
| 561 |
* filling them we'll switch to the beginning to finish. |
| 562 |
* If there is no info array we're doing a simple data |
| 563 |
* transfer request, whose opcode is IPA_CMD_NONE. |
| 564 |
*/ |
| 565 |
cmd_opcode = channel->command ? &trans->cmd_opcode[0] : NULL; |
| 566 |
avail = tre_ring->count - tre_ring->index % tre_ring->count; |
| 567 |
dest_tre = gsi_ring_virt(tre_ring, tre_ring->index); |
| 568 |
for_each_sg(trans->sgl, sg, trans->used_count, i) { |
| 569 |
bool last_tre = i == trans->used_count - 1; |
| 570 |
dma_addr_t addr = sg_dma_address(sg); |
| 571 |
u32 len = sg_dma_len(sg); |
| 572 |
|
| 573 |
byte_count += len; |
| 574 |
if (!avail--) |
| 575 |
dest_tre = gsi_ring_virt(tre_ring, 0); |
| 576 |
if (cmd_opcode) |
| 577 |
opcode = *cmd_opcode++; |
| 578 |
|
| 579 |
gsi_trans_tre_fill(dest_tre, addr, len, last_tre, bei, opcode); |
| 580 |
dest_tre++; |
| 581 |
} |
| 582 |
/* Associate the TRE with the transaction */ |
| 583 |
gsi_trans_map(trans, tre_ring->index); |
| 584 |
|
| 585 |
tre_ring->index += trans->used_count; |
| 586 |
|
| 587 |
trans->len = byte_count; |
| 588 |
if (channel->toward_ipa) |
| 589 |
gsi_trans_tx_committed(trans); |
| 590 |
|
| 591 |
gsi_trans_move_committed(trans); |
| 592 |
|
| 593 |
/* Ring doorbell if requested, or if all TREs are allocated */ |
| 594 |
if (ring_db || !atomic_read(&channel->trans_info.tre_avail)) { |
| 595 |
/* Report what we're handing off to hardware for TX channels */ |
| 596 |
if (channel->toward_ipa) |
| 597 |
gsi_trans_tx_queued(trans); |
| 598 |
gsi_trans_move_pending(trans); |
| 599 |
gsi_channel_doorbell(channel); |
| 600 |
} |
| 601 |
} |
| 602 |
|
| 603 |
/* Commit a GSI transaction */ |
| 604 |
void gsi_trans_commit(struct gsi_trans *trans, bool ring_db) |
| 605 |
{ |
| 606 |
if (trans->used_count) |
| 607 |
__gsi_trans_commit(trans, ring_db); |
| 608 |
else |
| 609 |
gsi_trans_free(trans); |
| 610 |
} |
| 611 |
|
| 612 |
/* Commit a GSI transaction and wait for it to complete */ |
| 613 |
void gsi_trans_commit_wait(struct gsi_trans *trans) |
| 614 |
{ |
| 615 |
if (!trans->used_count) |
| 616 |
goto out_trans_free; |
| 617 |
|
| 618 |
refcount_inc(&trans->refcount); |
| 619 |
|
| 620 |
__gsi_trans_commit(trans, true); |
| 621 |
|
| 622 |
wait_for_completion(&trans->completion); |
| 623 |
|
| 624 |
out_trans_free: |
| 625 |
gsi_trans_free(trans); |
| 626 |
} |
| 627 |
|
| 628 |
/* Process the completion of a transaction; called while polling */ |
| 629 |
void gsi_trans_complete(struct gsi_trans *trans) |
| 630 |
{ |
| 631 |
/* If the entire SGL was mapped when added, unmap it now */ |
| 632 |
if (trans->direction != DMA_NONE) |
| 633 |
dma_unmap_sg(trans->gsi->dev, trans->sgl, trans->used_count, |
| 634 |
trans->direction); |
| 635 |
|
| 636 |
ipa_gsi_trans_complete(trans); |
| 637 |
|
| 638 |
complete(&trans->completion); |
| 639 |
|
| 640 |
gsi_trans_free(trans); |
| 641 |
} |
| 642 |
|
| 643 |
/* Cancel a channel's pending transactions */ |
| 644 |
void gsi_channel_trans_cancel_pending(struct gsi_channel *channel) |
| 645 |
{ |
| 646 |
struct gsi_trans_info *trans_info = &channel->trans_info; |
| 647 |
u16 trans_id = trans_info->pending_id; |
| 648 |
|
| 649 |
/* channel->gsi->mutex is held by caller */ |
| 650 |
|
| 651 |
/* If there are no pending transactions, we're done */ |
| 652 |
if (trans_id == trans_info->committed_id) |
| 653 |
return; |
| 654 |
|
| 655 |
/* Mark all pending transactions cancelled */ |
| 656 |
do { |
| 657 |
struct gsi_trans *trans; |
| 658 |
|
| 659 |
trans = &trans_info->trans[trans_id % channel->tre_count]; |
| 660 |
trans->cancelled = true; |
| 661 |
} while (++trans_id != trans_info->committed_id); |
| 662 |
|
| 663 |
/* All pending transactions are now completed */ |
| 664 |
trans_info->pending_id = trans_info->committed_id; |
| 665 |
|
| 666 |
/* Schedule NAPI polling to complete the cancelled transactions */ |
| 667 |
napi_schedule(&channel->napi); |
| 668 |
} |
| 669 |
|
| 670 |
/* Issue a command to read a single byte from a channel */ |
| 671 |
int gsi_trans_read_byte(struct gsi *gsi, u32 channel_id, dma_addr_t addr) |
| 672 |
{ |
| 673 |
struct gsi_channel *channel = &gsi->channel[channel_id]; |
| 674 |
struct gsi_ring *tre_ring = &channel->tre_ring; |
| 675 |
struct gsi_trans_info *trans_info; |
| 676 |
struct gsi_tre *dest_tre; |
| 677 |
|
| 678 |
trans_info = &channel->trans_info; |
| 679 |
|
| 680 |
/* First reserve the TRE, if possible */ |
| 681 |
if (!gsi_trans_tre_reserve(trans_info, 1)) |
| 682 |
return -EBUSY; |
| 683 |
|
| 684 |
/* Now fill the reserved TRE and tell the hardware */ |
| 685 |
|
| 686 |
dest_tre = gsi_ring_virt(tre_ring, tre_ring->index); |
| 687 |
gsi_trans_tre_fill(dest_tre, addr, 1, true, false, IPA_CMD_NONE); |
| 688 |
|
| 689 |
tre_ring->index++; |
| 690 |
gsi_channel_doorbell(channel); |
| 691 |
|
| 692 |
return 0; |
| 693 |
} |
| 694 |
|
| 695 |
/* Mark a gsi_trans_read_byte() request done */ |
| 696 |
void gsi_trans_read_byte_done(struct gsi *gsi, u32 channel_id) |
| 697 |
{ |
| 698 |
struct gsi_channel *channel = &gsi->channel[channel_id]; |
| 699 |
|
| 700 |
gsi_trans_tre_release(&channel->trans_info, 1); |
| 701 |
} |
| 702 |
|
| 703 |
/* Initialize a channel's GSI transaction info */ |
| 704 |
int gsi_channel_trans_init(struct gsi *gsi, u32 channel_id) |
| 705 |
{ |
| 706 |
struct gsi_channel *channel = &gsi->channel[channel_id]; |
| 707 |
u32 tre_count = channel->tre_count; |
| 708 |
struct gsi_trans_info *trans_info; |
| 709 |
u32 tre_max; |
| 710 |
int ret; |
| 711 |
|
| 712 |
/* Ensure the size of a channel element is what's expected */ |
| 713 |
BUILD_BUG_ON(sizeof(struct gsi_tre) != GSI_RING_ELEMENT_SIZE); |
| 714 |
|
| 715 |
trans_info = &channel->trans_info; |
| 716 |
|
| 717 |
/* The tre_avail field is what ultimately limits the number of |
| 718 |
* outstanding transactions and their resources. A transaction |
| 719 |
* allocation succeeds only if the TREs available are sufficient |
| 720 |
* for what the transaction might need. |
| 721 |
*/ |
| 722 |
tre_max = gsi_channel_tre_max(channel->gsi, channel_id); |
| 723 |
atomic_set(&trans_info->tre_avail, tre_max); |
| 724 |
|
| 725 |
/* We can't use more TREs than the number available in the ring. |
| 726 |
* This limits the number of transactions that can be outstanding. |
| 727 |
* Worst case is one TRE per transaction (but we actually limit |
| 728 |
* it to something a little less than that). By allocating a |
| 729 |
* power-of-two number of transactions we can use an index |
| 730 |
* modulo that number to determine the next one that's free. |
| 731 |
* Transactions are allocated one at a time. |
| 732 |
*/ |
| 733 |
trans_info->trans = kzalloc_objs(*trans_info->trans, tre_count); |
| 734 |
if (!trans_info->trans) |
| 735 |
return -ENOMEM; |
| 736 |
trans_info->free_id = 0; /* all modulo channel->tre_count */ |
| 737 |
trans_info->allocated_id = 0; |
| 738 |
trans_info->committed_id = 0; |
| 739 |
trans_info->pending_id = 0; |
| 740 |
trans_info->completed_id = 0; |
| 741 |
trans_info->polled_id = 0; |
| 742 |
|
| 743 |
/* A completion event contains a pointer to the TRE that caused |
| 744 |
* the event (which will be the last one used by the transaction). |
| 745 |
* Each entry in this map records the transaction associated |
| 746 |
* with a corresponding completed TRE. |
| 747 |
*/ |
| 748 |
trans_info->map = kzalloc_objs(*trans_info->map, tre_count); |
| 749 |
if (!trans_info->map) { |
| 750 |
ret = -ENOMEM; |
| 751 |
goto err_trans_free; |
| 752 |
} |
| 753 |
|
| 754 |
/* A transaction uses a scatterlist array to represent the data |
| 755 |
* transfers implemented by the transaction. Each scatterlist |
| 756 |
* element is used to fill a single TRE when the transaction is |
| 757 |
* committed. So we need as many scatterlist elements as the |
| 758 |
* maximum number of TREs that can be outstanding. |
| 759 |
*/ |
| 760 |
ret = gsi_trans_pool_init(&trans_info->sg_pool, |
| 761 |
sizeof(struct scatterlist), |
| 762 |
tre_max, channel->trans_tre_max); |
| 763 |
if (ret) |
| 764 |
goto err_map_free; |
| 765 |
|
| 766 |
|
| 767 |
return 0; |
| 768 |
|
| 769 |
err_map_free: |
| 770 |
kfree(trans_info->map); |
| 771 |
err_trans_free: |
| 772 |
kfree(trans_info->trans); |
| 773 |
|
| 774 |
dev_err(gsi->dev, "error %d initializing channel %u transactions\n", |
| 775 |
ret, channel_id); |
| 776 |
|
| 777 |
return ret; |
| 778 |
} |
| 779 |
|
| 780 |
/* Inverse of gsi_channel_trans_init() */ |
| 781 |
void gsi_channel_trans_exit(struct gsi_channel *channel) |
| 782 |
{ |
| 783 |
struct gsi_trans_info *trans_info = &channel->trans_info; |
| 784 |
|
| 785 |
gsi_trans_pool_exit(&trans_info->sg_pool); |
| 786 |
kfree(trans_info->trans); |
| 787 |
kfree(trans_info->map); |
| 788 |
} |