Debian Codegraph @

prov/efa/test/efa_unit_test_ope.c

raw
1 /* SPDX-License-Identifier: BSD-2-Clause OR GPL-2.0-only */
2 /* SPDX-FileCopyrightText: Copyright Amazon.com, Inc. or its affiliates. All rights reserved. */
3
4 #include "efa_unit_tests.h"
5
6 typedef void (*efa_rdm_ope_handle_error_func_t)(struct efa_rdm_ope *ope, int err, int prov_errno);
7
8 void test_efa_rdm_ope_prepare_to_post_send_impl(struct efa_resource *resource,
9 enum fi_hmem_iface iface,
10 size_t total_len,
11 int expected_ret,
12 int expected_pkt_entry_cnt,
13 int *expected_pkt_entry_data_size_vec)
14 {
15 struct efa_ep_addr raw_addr;
16 struct efa_mr mock_mr;
17 struct efa_rdm_ope mock_txe;
18 struct efa_rdm_peer mock_peer;
19 size_t raw_addr_len = sizeof(raw_addr);
20 fi_addr_t addr;
21 int pkt_entry_cnt, pkt_entry_data_size_vec[1024];
22 int i, err, ret;
23
24 ret = fi_getname(&resource->ep->fid, &raw_addr, &raw_addr_len);
25 assert_int_equal(ret, 0);
26 raw_addr.qpn = 1;
27 raw_addr.qkey = 0x1234;
28 ret = fi_av_insert(resource->av, &raw_addr, 1, &addr, 0 /* flags */, NULL /* context */);
29 assert_int_equal(ret, 1);
30
31 mock_mr.peer.iface = iface;
32
33 memset(&mock_txe, 0, sizeof(mock_txe));
34 mock_txe.total_len = total_len;
35 mock_txe.addr = addr;
36 mock_txe.iov_count = 1;
37 mock_txe.iov[0].iov_base = NULL;
38 mock_txe.iov[0].iov_len = 9000;
39 mock_txe.desc[0] = &mock_mr;
40 mock_txe.ep = container_of(resource->ep, struct efa_rdm_ep, base_ep.util_ep.ep_fid);
41 mock_txe.peer = &mock_peer;
42
43 err = efa_rdm_ope_prepare_to_post_send(&mock_txe,
44 EFA_RDM_MEDIUM_MSGRTM_PKT,
45 &pkt_entry_cnt,
46 pkt_entry_data_size_vec);
47
48 assert_int_equal(err, expected_ret);
49
50 if (err)
51 return;
52
53 assert_int_equal(pkt_entry_cnt, expected_pkt_entry_cnt);
54
55 for (i = 0; i < pkt_entry_cnt; ++i)
56 assert_int_equal(pkt_entry_data_size_vec[i], expected_pkt_entry_data_size_vec[i]);
57 }
58
59 /**
60 * @brief verify efa_rdm_ope_prepare_to_post_send()'s return code
61 *
62 * Verify that efa_rdm_ope_prepare_to_post_send() will return
63 * -FI_EAGAIN, when there is not enough TX packet available,
64 */
65 void test_efa_rdm_ope_prepare_to_post_send_with_no_enough_tx_pkts(struct efa_resource **state)
66 {
67 struct efa_resource *resource = *state;
68 struct efa_rdm_ep *efa_rdm_ep;
69
70 efa_unit_test_resource_construct(resource, FI_EP_RDM, EFA_FABRIC_NAME);
71
72 efa_rdm_ep = container_of(resource->ep, struct efa_rdm_ep, base_ep.util_ep.ep_fid);
73 efa_rdm_ep->efa_outstanding_tx_ops = efa_rdm_ep->efa_max_outstanding_tx_ops - 1;
74 /* we need at least 2 packets to send this message, but only 1 is available,
75 * therefore efa_rdm_ope_prepare_to_post_send() should return
76 * -FI_EAGAIN.
77 */
78 test_efa_rdm_ope_prepare_to_post_send_impl(resource, FI_HMEM_SYSTEM,
79 9000, -FI_EAGAIN, -1, NULL);
80 efa_rdm_ep->efa_outstanding_tx_ops = 0;
81 }
82
83 /**
84 * @brief verify the pkt_entry_cnt and data size for host memory
85 */
86 void test_efa_rdm_ope_prepare_to_post_send_host_memory(struct efa_resource **state)
87 {
88 struct efa_resource *resource = *state;
89 size_t msg_length;
90 int expected_pkt_entry_cnt;
91 int expected_pkt_entry_data_size_vec[1024];
92
93 efa_unit_test_resource_construct(resource, FI_EP_RDM, EFA_FABRIC_NAME);
94
95 /* data size should be aligned and evenly distributed.
96 * alignment for host memory is 8 byte by default.
97 */
98 msg_length = 9000;
99 expected_pkt_entry_cnt = 2;
100 expected_pkt_entry_data_size_vec[0] = 4496;
101 expected_pkt_entry_data_size_vec[1] = 4504;
102 test_efa_rdm_ope_prepare_to_post_send_impl(resource, FI_HMEM_SYSTEM,
103 msg_length,
104 0,
105 expected_pkt_entry_cnt,
106 expected_pkt_entry_data_size_vec);
107
108 msg_length = 12000;
109 expected_pkt_entry_cnt = 2;
110 expected_pkt_entry_data_size_vec[0] = 6000;
111 expected_pkt_entry_data_size_vec[1] = 6000;
112 test_efa_rdm_ope_prepare_to_post_send_impl(resource, FI_HMEM_SYSTEM,
113 msg_length,
114 0,
115 expected_pkt_entry_cnt,
116 expected_pkt_entry_data_size_vec);
117
118 msg_length = 18004;
119 expected_pkt_entry_cnt = 3;
120 expected_pkt_entry_data_size_vec[0] = 6000;
121 expected_pkt_entry_data_size_vec[1] = 6000;
122 expected_pkt_entry_data_size_vec[2] = 6004;
123 test_efa_rdm_ope_prepare_to_post_send_impl(resource, FI_HMEM_SYSTEM,
124 msg_length,
125 0,
126 expected_pkt_entry_cnt,
127 expected_pkt_entry_data_size_vec);
128
129 }
130
131 /**
132 * @brief verify the pkt_entry_cnt and data size for host memory when align128 was requested
133 */
134 void test_efa_rdm_ope_prepare_to_post_send_host_memory_align128(struct efa_resource **state)
135 {
136 struct efa_resource *resource = *state;
137 struct efa_rdm_ep *efa_rdm_ep;
138 size_t msg_length;
139 int expected_pkt_entry_cnt;
140 int expected_pkt_entry_data_size_vec[1024];
141
142 efa_unit_test_resource_construct(resource, FI_EP_RDM, EFA_FABRIC_NAME);
143 efa_rdm_ep = container_of(resource->ep, struct efa_rdm_ep, base_ep.util_ep.ep_fid);
144 efa_rdm_ep->sendrecv_in_order_aligned_128_bytes = true;
145
146 /* if user requested 128 byte alignment, then all but the last
147 * last packet's data size should be 128 aligned
148 */
149 msg_length = 9000;
150 expected_pkt_entry_cnt = 2;
151 expected_pkt_entry_data_size_vec[0] = 4480;
152 expected_pkt_entry_data_size_vec[1] = 4520;
153 test_efa_rdm_ope_prepare_to_post_send_impl(resource, FI_HMEM_SYSTEM,
154 msg_length,
155 0,
156 expected_pkt_entry_cnt,
157 expected_pkt_entry_data_size_vec);
158
159 msg_length = 12000;
160 expected_pkt_entry_cnt = 2;
161 expected_pkt_entry_data_size_vec[0] = 5888;
162 expected_pkt_entry_data_size_vec[1] = 6112;
163 test_efa_rdm_ope_prepare_to_post_send_impl(resource, FI_HMEM_SYSTEM,
164 msg_length,
165 0,
166 expected_pkt_entry_cnt,
167 expected_pkt_entry_data_size_vec);
168
169 msg_length = 18004;
170 expected_pkt_entry_cnt = 3;
171 expected_pkt_entry_data_size_vec[0] = 5888;
172 expected_pkt_entry_data_size_vec[1] = 5888;
173 expected_pkt_entry_data_size_vec[2] = 6228;
174 test_efa_rdm_ope_prepare_to_post_send_impl(resource, FI_HMEM_SYSTEM,
175 msg_length,
176 0,
177 expected_pkt_entry_cnt,
178 expected_pkt_entry_data_size_vec);
179 }
180
181 /**
182 * @brief verify the pkt_entry_cnt and data size for cuda memory
183 */
184 void test_efa_rdm_ope_prepare_to_post_send_cuda_memory(struct efa_resource **state)
185 {
186 struct efa_resource *resource = *state;
187 size_t msg_length;
188 int expected_pkt_entry_cnt;
189 int expected_pkt_entry_data_size_vec[1024];
190
191 efa_unit_test_resource_construct(resource, FI_EP_RDM, EFA_FABRIC_NAME);
192
193 /* default alignment of cuda memory is 64 bytes */
194 msg_length = 12000;
195 expected_pkt_entry_cnt = 2;
196 expected_pkt_entry_data_size_vec[0] = 5952;
197 expected_pkt_entry_data_size_vec[1] = 6048;
198 test_efa_rdm_ope_prepare_to_post_send_impl(resource, FI_HMEM_CUDA,
199 msg_length,
200 0,
201 expected_pkt_entry_cnt,
202 expected_pkt_entry_data_size_vec);
203 }
204
205 /**
206 * @brief verify the pkt_entry_cnt and data size for cuda memory when align128 was requested
207 */
208 void test_efa_rdm_ope_prepare_to_post_send_cuda_memory_align128(struct efa_resource **state)
209 {
210 struct efa_resource *resource = *state;
211 struct efa_rdm_ep *efa_rdm_ep;
212 size_t msg_length;
213 int expected_pkt_entry_cnt;
214 int expected_pkt_entry_data_size_vec[1024];
215
216 efa_unit_test_resource_construct(resource, FI_EP_RDM, EFA_FABRIC_NAME);
217 efa_rdm_ep = container_of(resource->ep, struct efa_rdm_ep, base_ep.util_ep.ep_fid);
218 efa_rdm_ep->sendrecv_in_order_aligned_128_bytes = true;
219
220 msg_length = 12000;
221 expected_pkt_entry_cnt = 2;
222 /* if user requested 128 byte alignment, then all but the last
223 * last packet's data size should be 128 aligned
224 */
225 expected_pkt_entry_data_size_vec[0] = 5888;
226 expected_pkt_entry_data_size_vec[1] = 6112;
227 test_efa_rdm_ope_prepare_to_post_send_impl(resource, FI_HMEM_CUDA,
228 msg_length,
229 0,
230 expected_pkt_entry_cnt,
231 expected_pkt_entry_data_size_vec);
232 }
233
234 /**
235 * @brief verify that 0 byte write can be submitted successfully
236 */
237 void test_efa_rdm_ope_post_write_0_byte(struct efa_resource **state)
238 {
239 struct efa_resource *resource = *state;
240 struct ibv_qp_ex *ibv_qpx;
241 struct efa_unit_test_buff local_buff;
242 struct efa_ep_addr raw_addr;
243 struct efa_rdm_ope mock_txe;
244 size_t raw_addr_len = sizeof(raw_addr);
245 fi_addr_t addr;
246 int ret, err;
247
248 efa_unit_test_resource_construct(resource, FI_EP_RDM, EFA_FABRIC_NAME);
249
250 ret = fi_getname(&resource->ep->fid, &raw_addr, &raw_addr_len);
251 assert_int_equal(ret, 0);
252 raw_addr.qpn = 1;
253 raw_addr.qkey = 0x1234;
254 ret = fi_av_insert(resource->av, &raw_addr, 1, &addr, 0 /* flags */, NULL /* context */);
255 assert_int_equal(ret, 1);
256
257 efa_unit_test_buff_construct(&local_buff, resource, 4096 /* buff_size */);
258 memset(&mock_txe, 0, sizeof(mock_txe));
259 mock_txe.total_len = 0;
260 mock_txe.addr = addr;
261 mock_txe.iov_count = 1;
262 mock_txe.iov[0].iov_base = local_buff.buff;
263 mock_txe.iov[0].iov_len = 0;
264 mock_txe.desc[0] = fi_mr_desc(local_buff.mr);
265 mock_txe.rma_iov_count = 1;
266 mock_txe.rma_iov[0].addr = 0x87654321;
267 mock_txe.rma_iov[0].key = 123456;
268 mock_txe.rma_iov[0].len = 0;
269
270 mock_txe.ep = container_of(resource->ep, struct efa_rdm_ep, base_ep.util_ep.ep_fid);
271
272 ibv_qpx = mock_txe.ep->base_ep.qp->ibv_qp_ex;
273 ibv_qpx->wr_start = &efa_mock_ibv_wr_start_no_op;
274 ibv_qpx->wr_rdma_write = &efa_mock_ibv_wr_rdma_write_save_wr;
275 ibv_qpx->wr_set_sge_list = &efa_mock_ibv_wr_set_sge_list_no_op;
276 ibv_qpx->wr_set_ud_addr = &efa_mock_ibv_wr_set_ud_addr_no_op;
277 ibv_qpx->wr_complete = &efa_mock_ibv_wr_complete_no_op;
278
279 assert_int_equal(g_ibv_submitted_wr_id_cnt, 0);
280 err = efa_rdm_ope_post_remote_write(&mock_txe);
281 assert_int_equal(err, 0);
282 assert_int_equal(g_ibv_submitted_wr_id_cnt, 1);
283
284 efa_rdm_pke_release_tx((struct efa_rdm_pke *)g_ibv_submitted_wr_id_vec[0]);
285 mock_txe.ep->efa_outstanding_tx_ops = 0;
286 efa_unit_test_buff_destruct(&local_buff);
287 }
288
289 /**
290 * @brief efa_rdm_rxe_post_local_read_or_queue should call
291 * efa_rdm_pke_read.
292 * When efa_rdm_pke_read failed,
293 * make sure there is no txe leak in efa_rdm_rxe_post_local_read_or_queue
294 *
295 * @param[in] state struct efa_resource that is managed by the framework
296 * @param[in] efa_rdm_pke_read_return return code of efa_rdm_pke_read
297 */
298 static
299 void test_efa_rdm_rxe_post_local_read_or_queue_impl(struct efa_resource *resource, int efa_rdm_pke_read_return)
300 {
301 struct efa_rdm_ep *efa_rdm_ep;
302 struct efa_rdm_pke *pkt_entry;
303 struct efa_rdm_ope *rxe;
304 struct efa_mr cuda_mr = {0};
305 char buf[16];
306 struct iovec iov = {
307 .iov_base = buf,
308 .iov_len = sizeof buf
309 };
310
311 /**
312 * TODO: Ideally we should mock efa_rdm_ope_post_remote_read_or_queue here,
313 * but this function is currently cannot be mocked as it is at the same file
314 * with efa_rdm_rxe_post_local_read_or_queue, see this restriction in
315 * prov/efa/test/README.md's mocking session
316 */
317 g_efa_unit_test_mocks.efa_rdm_pke_read = &efa_mock_efa_rdm_pke_read_return_mock;
318
319 efa_rdm_ep = container_of(resource->ep, struct efa_rdm_ep, base_ep.util_ep.ep_fid);
320
321 /* Fake a rdma read enabled device */
322 efa_rdm_ep_domain(efa_rdm_ep)->device->max_rdma_size = efa_env.efa_read_segment_size;
323
324 pkt_entry = efa_rdm_pke_alloc(efa_rdm_ep, efa_rdm_ep->efa_rx_pkt_pool, EFA_RDM_PKE_FROM_EFA_RX_POOL);
325 assert_non_null(pkt_entry);
326 pkt_entry->payload = pkt_entry->wiredata;
327
328 rxe = efa_rdm_ep_alloc_rxe(efa_rdm_ep, FI_ADDR_UNSPEC, ofi_op_tagged);
329 cuda_mr.peer.iface = FI_HMEM_CUDA;
330
331 rxe->desc[0] = &cuda_mr;
332 rxe->iov_count = 1;
333 rxe->iov[0] = iov;
334 pkt_entry->ope = rxe;
335
336 assert_true(dlist_empty(&efa_rdm_ep->txe_list));
337
338 will_return(efa_mock_efa_rdm_pke_read_return_mock, efa_rdm_pke_read_return);
339
340 assert_int_equal(efa_rdm_rxe_post_local_read_or_queue(rxe, 0, pkt_entry, pkt_entry->payload, 16), efa_rdm_pke_read_return);
341
342 /* Clean up the rx entry no matter what returns */
343 efa_rdm_pke_release_rx(pkt_entry);
344 }
345
346 void test_efa_rdm_rxe_post_local_read_or_queue_unhappy(struct efa_resource **state)
347 {
348 struct efa_rdm_ep *efa_rdm_ep;
349 struct efa_resource *resource = *state;
350
351 efa_unit_test_resource_construct(resource, FI_EP_RDM, EFA_FABRIC_NAME);
352
353 test_efa_rdm_rxe_post_local_read_or_queue_impl(resource, -FI_ENOMR);
354
355 efa_rdm_ep = container_of(resource->ep, struct efa_rdm_ep, base_ep.util_ep.ep_fid);
356
357 /* Make sure txe is cleaned for a failed read */
358 assert_true(dlist_empty(&efa_rdm_ep->txe_list));
359 }
360
361 void test_efa_rdm_rxe_post_local_read_or_queue_happy(struct efa_resource **state)
362 {
363 struct efa_rdm_ep *efa_rdm_ep;
364 struct efa_resource *resource = *state;
365 struct efa_rdm_pke *tx_pkt_entry;
366 struct efa_rdm_ope *txe;
367
368 efa_unit_test_resource_construct(resource, FI_EP_RDM, EFA_FABRIC_NAME);
369
370 test_efa_rdm_rxe_post_local_read_or_queue_impl(resource, FI_SUCCESS);
371
372 efa_rdm_ep = container_of(resource->ep, struct efa_rdm_ep, base_ep.util_ep.ep_fid);
373 /* Now we should have a txe allocated */
374 assert_int_equal(efa_unit_test_get_dlist_length(&efa_rdm_ep->txe_list), 1);
375 txe = container_of(efa_rdm_ep->txe_list.next, struct efa_rdm_ope, ep_entry);
376 assert_true(txe->internal_flags & EFA_RDM_OPE_INTERNAL);
377
378 /* We also have a tx pkt allocated inside efa_rdm_ope_read
379 * and we need to clean it */
380 tx_pkt_entry = ofi_bufpool_get_ibuf(efa_rdm_ep->efa_tx_pkt_pool, 0);
381 efa_rdm_pke_release(tx_pkt_entry);
382 }
383
384 static
385 void test_efa_rdm_ope_handle_error_impl(
386 struct efa_resource *resource,
387 efa_rdm_ope_handle_error_func_t efa_rdm_ope_handle_error,
388 struct efa_rdm_ope *ope, bool expect_cq_error)
389 {
390 struct fi_cq_data_entry cq_entry;
391 struct fi_cq_err_entry cq_err_entry = {0};
392 struct fi_eq_err_entry eq_err_entry;
393
394 efa_rdm_ope_handle_error(ope, FI_ENOTCONN,
395 EFA_IO_COMP_STATUS_LOCAL_ERROR_UNREACH_REMOTE);
396
397 if (expect_cq_error) {
398 assert_int_equal(fi_cq_read(resource->cq, &cq_entry, 1),
399 -FI_EAVAIL);
400 assert_int_equal(fi_cq_readerr(resource->cq, &cq_err_entry, 0),
401 1);
402 assert_int_equal(cq_err_entry.err, FI_ENOTCONN);
403 assert_int_equal(cq_err_entry.prov_errno,
404 EFA_IO_COMP_STATUS_LOCAL_ERROR_UNREACH_REMOTE);
405 } else {
406 /* We should expect an empty cq and an eq error */
407 assert_int_equal(fi_cq_read(resource->cq, &cq_entry, 1),
408 -FI_EAGAIN);
409 assert_int_equal(fi_eq_readerr(resource->eq, &eq_err_entry, 0),
410 sizeof(eq_err_entry));
411 assert_int_equal(eq_err_entry.err, FI_ENOTCONN);
412 assert_int_equal(eq_err_entry.prov_errno,
413 EFA_IO_COMP_STATUS_LOCAL_ERROR_UNREACH_REMOTE);
414 }
415 }
416
417 void test_efa_rdm_txe_handle_error_write_cq(struct efa_resource **state)
418 {
419 struct efa_resource *resource = *state;
420 struct efa_rdm_ope *txe;
421
422 efa_unit_test_resource_construct(resource, FI_EP_RDM, EFA_FABRIC_NAME);
423
424 txe = efa_unit_test_alloc_txe(resource, ofi_op_write);
425 assert_non_null(txe);
426
427 test_efa_rdm_ope_handle_error_impl(resource, efa_rdm_txe_handle_error, txe, true);
428
429 efa_rdm_txe_release(txe);
430 }
431
432 void test_efa_rdm_txe_handle_error_not_write_cq(struct efa_resource **state)
433 {
434 struct efa_resource *resource = *state;
435 struct efa_rdm_ope *txe;
436
437 efa_unit_test_resource_construct(resource, FI_EP_RDM, EFA_FABRIC_NAME);
438
439 txe = efa_unit_test_alloc_txe(resource, ofi_op_write);
440 assert_non_null(txe);
441
442 txe->internal_flags |= EFA_RDM_OPE_INTERNAL;
443
444 test_efa_rdm_ope_handle_error_impl(resource, efa_rdm_txe_handle_error, txe, false);
445
446 efa_rdm_txe_release(txe);
447 }
448
449 void test_efa_rdm_rxe_handle_error_write_cq(struct efa_resource **state)
450 {
451 struct efa_resource *resource = *state;
452 struct efa_rdm_ope *rxe;
453
454 efa_unit_test_resource_construct(resource, FI_EP_RDM, EFA_FABRIC_NAME);
455
456 rxe = efa_unit_test_alloc_rxe(resource, ofi_op_tagged);
457 assert_non_null(rxe);
458
459 test_efa_rdm_ope_handle_error_impl(resource, efa_rdm_rxe_handle_error, rxe, true);
460
461 efa_rdm_rxe_release(rxe);
462 }
463
464 void test_efa_rdm_rxe_handle_error_not_write_cq(struct efa_resource **state)
465 {
466 struct efa_resource *resource = *state;
467 struct efa_rdm_ope *rxe;
468
469 efa_unit_test_resource_construct(resource, FI_EP_RDM, EFA_FABRIC_NAME);
470
471 rxe = efa_unit_test_alloc_rxe(resource, ofi_op_tagged);
472 assert_non_null(rxe);
473
474 rxe->internal_flags |= EFA_RDM_OPE_INTERNAL;
475
476 test_efa_rdm_ope_handle_error_impl(resource, efa_rdm_rxe_handle_error, rxe, false);
477
478 efa_rdm_rxe_release(rxe);
479 }
480
481 void test_efa_rdm_rxe_map(struct efa_resource **state)
482 {
483 struct efa_resource *resource = *state;
484 struct efa_rdm_ope *rxe;
485 struct efa_rdm_ep *efa_rdm_ep;
486
487 efa_unit_test_resource_construct(resource, FI_EP_RDM, EFA_FABRIC_NAME);
488
489 rxe = efa_unit_test_alloc_rxe(resource, ofi_op_tagged);
490 rxe->msg_id = 1;
491 assert_non_null(rxe);
492
493 /* rxe has not been inserted to any rxe_map yet */
494 assert_null(rxe->rxe_map);
495
496 efa_rdm_ep = container_of(resource->ep, struct efa_rdm_ep,
497 base_ep.util_ep.ep_fid);
498
499 efa_rdm_rxe_map_insert(&efa_rdm_ep->rxe_map, rxe->msg_id, rxe->addr,
500 rxe);
501 assert_true(rxe->rxe_map == &efa_rdm_ep->rxe_map);
502 assert_true(rxe == efa_rdm_rxe_map_lookup(rxe->rxe_map, rxe->msg_id,
503 rxe->addr));
504
505 efa_rdm_rxe_release(rxe);
506
507 /**
508 * Now the map_entry_pool should be empty so we can destroy it
509 * Otherwise there will be an assertion error on the use cnt is
510 * is non-zero
511 */
512 ofi_bufpool_destroy(efa_rdm_ep->map_entry_pool);
513 efa_rdm_ep->map_entry_pool = NULL;
514 }