Debian Codegraph @

prov/efa/test/efa_unit_test_ep.c

raw
1 /* SPDX-License-Identifier: BSD-2-Clause OR GPL-2.0-only */
2 /* SPDX-FileCopyrightText: Copyright Amazon.com, Inc. or its affiliates. All rights reserved. */
3
4 #include "efa_unit_tests.h"
5 #include "rdm/efa_rdm_cq.h"
6 #include "efa_rdm_pke_utils.h"
7
8 /**
9 * @brief Verify the EFA RDM endpoint correctly parses the host id string
10 * @param[in] state cmocka state variable
11 * @param[in] file_exists Toggle whether the host id file exists
12 * @param[in] raw_id The host id string that is written in the host id file.
13 * @param[in] expect_id Expected parsed host id integer
14 */
15 void test_efa_rdm_ep_host_id(struct efa_resource **state, bool file_exists, char *raw_id, uint64_t expect_id)
16 {
17 int fd = -1;
18 ssize_t written_len;
19 char host_id_file[] = "XXXXXXXXXX";
20 struct efa_resource *resource = *state;
21 struct efa_rdm_ep *efa_rdm_ep;
22
23 efa_env.host_id_file = NULL;
24
25 if (file_exists) {
26 fd = mkstemp(host_id_file);
27 if (fd < 0) {
28 fail();
29 }
30
31 written_len = write(fd, raw_id, strlen(raw_id));
32 if (written_len != strlen(raw_id)) {
33 close(fd);
34 fail();
35 }
36
37 efa_env.host_id_file = host_id_file;
38 }
39
40 efa_unit_test_resource_construct(resource, FI_EP_RDM, EFA_FABRIC_NAME);
41
42 efa_rdm_ep = container_of(resource->ep, struct efa_rdm_ep, base_ep.util_ep.ep_fid);
43
44 /* Remove the temporary file */
45 if (efa_env.host_id_file) {
46 unlink(efa_env.host_id_file);
47 close(fd);
48 }
49
50 assert_int_equal(efa_rdm_ep->host_id, expect_id);
51 }
52
53 /**
54 * @brief Verify the EFA RDM endpoint ignores non-existent host id file
55 */
56 void test_efa_rdm_ep_ignore_missing_host_id_file(struct efa_resource **state)
57 {
58 test_efa_rdm_ep_host_id(state, false, NULL, 0);
59 }
60
61 /**
62 * @brief Verify the EFA RDM endpoint correctly parses a valid host id string
63 */
64 void test_efa_rdm_ep_has_valid_host_id(struct efa_resource **state)
65 {
66 test_efa_rdm_ep_host_id(state, true, "i-01234567812345678", 0x1234567812345678);
67 }
68
69 /**
70 * @brief Verify the EFA RDM endpoint ignores a short (<16 char) host id string
71 */
72 void test_efa_rdm_ep_ignore_short_host_id(struct efa_resource **state)
73 {
74 test_efa_rdm_ep_host_id(state, true, "i-012345678", 0);
75 }
76
77 /**
78 * @brief Verify the EFA RDM endpoint ignores a malformatted host id string
79 */
80 void test_efa_rdm_ep_ignore_non_hex_host_id(struct efa_resource **state)
81 {
82 test_efa_rdm_ep_host_id(state, true, "i-0abcdefghabcdefgh", 0);
83 }
84
85 #if HAVE_EFADV_CQ_EX
86 /**
87 * @brief Verify the EFA RDM endpoint correctly processes and responds to a handshake packet
88 * Upon receiving a handshake packet from a new remote peer, the endpoint should inspect
89 * the packet header and set the peer host id if HOST_ID_HDR is turned on.
90 * Then the endpoint should respond with a handshake packet, and include the local host id
91 * if and only if it is non-zero.
92 *
93 * @param[in] state cmocka state variable
94 * @param[in] local_host_id The local host id
95 * @param[in] peer_host_id The remote peer host id
96 * @param[in] include_connid Toggle whether connid should be included in handshake packet
97 */
98 void test_efa_rdm_ep_handshake_exchange_host_id(struct efa_resource **state, uint64_t local_host_id, uint64_t peer_host_id, bool include_connid)
99 {
100 fi_addr_t peer_addr = 0;
101 int cq_read_recv_ret, cq_read_send_ret;
102 struct efa_ep_addr raw_addr = {0};
103 size_t raw_addr_len = sizeof(raw_addr);
104 struct efa_rdm_peer *peer;
105 struct efa_resource *resource = *state;
106 struct efa_unit_test_handshake_pkt_attr pkt_attr = {0};
107 struct fi_cq_data_entry cq_entry;
108 struct ibv_qp_ex *ibv_qp;
109 struct efa_rdm_ep *efa_rdm_ep;
110 struct efa_rdm_pke *pkt_entry;
111 uint64_t actual_peer_host_id = UINT64_MAX;
112 struct efa_rdm_cq *efa_rdm_cq;
113 struct ibv_cq_ex *ibv_cqx;
114
115 g_efa_unit_test_mocks.local_host_id = local_host_id;
116 g_efa_unit_test_mocks.peer_host_id = peer_host_id;
117
118 assert_false(actual_peer_host_id == g_efa_unit_test_mocks.peer_host_id);
119
120 /* disable shm to force using efa device to send */
121 efa_unit_test_resource_construct_rdm_shm_disabled(resource);
122
123 efa_rdm_ep = container_of(resource->ep, struct efa_rdm_ep, base_ep.util_ep.ep_fid);
124 efa_rdm_cq = container_of(resource->cq, struct efa_rdm_cq, efa_cq.util_cq.cq_fid.fid);
125 ibv_cqx = efa_rdm_cq->efa_cq.ibv_cq.ibv_cq_ex;
126
127 efa_rdm_ep->host_id = g_efa_unit_test_mocks.local_host_id;
128
129 /* Create and register a fake peer */
130 assert_int_equal(fi_getname(&resource->ep->fid, &raw_addr, &raw_addr_len), 0);
131 raw_addr.qpn = 0;
132 raw_addr.qkey = 0x1234;
133
134 assert_int_equal(fi_av_insert(resource->av, &raw_addr, 1, &peer_addr, 0, NULL), 1);
135
136 peer = efa_rdm_ep_get_peer(efa_rdm_ep, peer_addr);
137 assert_non_null(peer);
138 /* Peer host id is uninitialized before handshake */
139 assert_int_equal(peer->host_id, 0);
140 assert_int_not_equal(peer->flags & EFA_RDM_PEER_HANDSHAKE_SENT, EFA_RDM_PEER_HANDSHAKE_SENT);
141
142 /*
143 * The rx pkt entry should only be allocated and posted by the progress engine.
144 * However, to mock a receive completion, we have to allocate an rx entry
145 * and modify it out of band. The proess engine grow the rx pool in the first
146 * call and set efa_rdm_ep->efa_rx_pkts_posted as the rx pool size. Here we
147 * follow the progress engine to set the efa_rx_pkts_posted counter manually
148 * TODO: modify the rx pkt as part of the ibv cq poll mock so we don't have to
149 * allocate pkt entry and hack the pkt counters.
150 */
151 pkt_entry = efa_rdm_pke_alloc(efa_rdm_ep, efa_rdm_ep->efa_rx_pkt_pool, EFA_RDM_PKE_FROM_EFA_RX_POOL);
152 assert_non_null(pkt_entry);
153 efa_rdm_ep->efa_rx_pkts_posted = efa_rdm_ep_get_rx_pool_size(efa_rdm_ep);
154
155 pkt_attr.connid = include_connid ? raw_addr.qkey : 0;
156 pkt_attr.host_id = g_efa_unit_test_mocks.peer_host_id;
157 pkt_attr.device_version = 0xefa0;
158 efa_unit_test_handshake_pkt_construct(pkt_entry, &pkt_attr);
159
160 ibv_qp = efa_rdm_ep->base_ep.qp->ibv_qp_ex;
161 ibv_qp->wr_start = &efa_mock_ibv_wr_start_no_op;
162 /* this mock will save the send work request (wr) in a global array */
163 ibv_qp->wr_send = &efa_mock_ibv_wr_send_verify_handshake_pkt_local_host_id_and_save_wr;
164 ibv_qp->wr_set_inline_data_list = &efa_mock_ibv_wr_set_inline_data_list_no_op;
165 ibv_qp->wr_set_sge_list = &efa_mock_ibv_wr_set_sge_list_no_op;
166 ibv_qp->wr_set_ud_addr = &efa_mock_ibv_wr_set_ud_addr_no_op;
167 ibv_qp->wr_complete = &efa_mock_ibv_wr_complete_no_op;
168 expect_function_call(efa_mock_ibv_wr_send_verify_handshake_pkt_local_host_id_and_save_wr);
169
170 /* Setup CQ */
171 ibv_cqx->end_poll = &efa_mock_ibv_end_poll_check_mock;
172 ibv_cqx->next_poll = &efa_mock_ibv_next_poll_check_function_called_and_return_mock;
173 ibv_cqx->read_byte_len = &efa_mock_ibv_read_byte_len_return_mock;
174 ibv_cqx->read_opcode = &efa_mock_ibv_read_opcode_return_mock;
175 ibv_cqx->read_slid = &efa_mock_ibv_read_slid_return_mock;
176 ibv_cqx->read_src_qp = &efa_mock_ibv_read_src_qp_return_mock;
177 ibv_cqx->read_qp_num = &efa_mock_ibv_read_qp_num_return_mock;
178 ibv_cqx->read_wc_flags = &efa_mock_ibv_read_wc_flags_return_mock;
179 ibv_cqx->read_vendor_err = &efa_mock_ibv_read_vendor_err_return_mock;
180 ibv_cqx->start_poll = &efa_mock_ibv_start_poll_return_mock;
181 ibv_cqx->status = IBV_WC_SUCCESS;
182 ibv_cqx->wr_id = (uintptr_t)pkt_entry;
183 expect_function_call(efa_mock_ibv_next_poll_check_function_called_and_return_mock);
184
185 /* Receive handshake packet */
186 will_return(efa_mock_ibv_end_poll_check_mock, NULL);
187 will_return(efa_mock_ibv_next_poll_check_function_called_and_return_mock, ENOENT);
188 will_return(efa_mock_ibv_read_byte_len_return_mock, pkt_entry->pkt_size);
189 will_return(efa_mock_ibv_read_opcode_return_mock, IBV_WC_RECV);
190 will_return(efa_mock_ibv_read_qp_num_return_mock, efa_rdm_ep->base_ep.qp->qp_num);
191 will_return(efa_mock_ibv_read_wc_flags_return_mock, 0);
192 will_return(efa_mock_ibv_read_slid_return_mock, efa_rdm_ep_get_peer_ahn(efa_rdm_ep, peer_addr));
193 will_return(efa_mock_ibv_read_src_qp_return_mock, raw_addr.qpn);
194 will_return(efa_mock_ibv_start_poll_return_mock, IBV_WC_SUCCESS);
195
196 /**
197 * Fire away handshake packet.
198 * Because we don't care if it fails(there is no receiver!), mark it as failed to make mocking simpler.
199 */
200 will_return(efa_mock_ibv_end_poll_check_mock, NULL);
201 will_return(efa_mock_ibv_read_opcode_return_mock, IBV_WC_SEND);
202 will_return(efa_mock_ibv_read_qp_num_return_mock, efa_rdm_ep->base_ep.qp->qp_num);
203 will_return(efa_mock_ibv_read_vendor_err_return_mock, FI_EFA_ERR_OTHER);
204 will_return(efa_mock_ibv_start_poll_return_mock, IBV_WC_SUCCESS);
205
206 /* Progress the recv wr first to process the received handshake packet. */
207 cq_read_recv_ret = fi_cq_read(resource->cq, &cq_entry, 1);
208
209 actual_peer_host_id = peer->host_id;
210
211 /**
212 * We need to poll the CQ twice explicitly to point the CQE
213 * to the saved send wr in handshake
214 */
215 ibv_cqx->status = IBV_WC_GENERAL_ERR;
216 ibv_cqx->wr_id = (uintptr_t)g_ibv_submitted_wr_id_vec[0];
217
218 /* Progress the send wr to clean up outstanding tx ops */
219 cq_read_send_ret = fi_cq_read(resource->cq, &cq_entry, 1);
220
221 /* HANDSHAKE packet does not generate completion entry */
222 assert_int_equal(cq_read_recv_ret, -FI_EAGAIN);
223 assert_int_equal(cq_read_send_ret, -FI_EAGAIN);
224
225 /* Peer host id is set after handshake */
226 assert_true(actual_peer_host_id == g_efa_unit_test_mocks.peer_host_id);
227
228 /* Device version should be stored after handshake */
229 assert_int_equal(peer->device_version, 0xefa0);
230 }
231 #else
232 void test_efa_rdm_ep_handshake_exchange_host_id() {
233 skip();
234 }
235 #endif
236
237 void test_efa_rdm_ep_handshake_receive_and_send_valid_host_ids_with_connid(struct efa_resource **state)
238 {
239 test_efa_rdm_ep_handshake_exchange_host_id(state, 0x1234567812345678, 0x8765432187654321, true);
240 }
241
242 void test_efa_rdm_ep_handshake_receive_and_send_valid_host_ids_without_connid(struct efa_resource **state)
243 {
244 test_efa_rdm_ep_handshake_exchange_host_id(state, 0x1234567812345678, 0x8765432187654321, false);
245 }
246
247 void test_efa_rdm_ep_handshake_receive_valid_peer_host_id_and_do_not_send_local_host_id(struct efa_resource **state)
248 {
249 test_efa_rdm_ep_handshake_exchange_host_id(state, 0x0, 0x8765432187654321, true);
250 }
251
252 void test_efa_rdm_ep_handshake_receive_without_peer_host_id_and_do_not_send_local_host_id(struct efa_resource **state)
253 {
254 test_efa_rdm_ep_handshake_exchange_host_id(state, 0x0, 0x0, true);
255 }
256
257 static void check_ep_pkt_pool_flags(struct fid_ep *ep, int expected_flags)
258 {
259 struct efa_rdm_ep *efa_rdm_ep;
260
261 efa_rdm_ep = container_of(ep, struct efa_rdm_ep, base_ep.util_ep.ep_fid);
262 assert_int_equal(efa_rdm_ep->efa_tx_pkt_pool->attr.flags, expected_flags);
263 assert_int_equal(efa_rdm_ep->efa_rx_pkt_pool->attr.flags, expected_flags);
264 }
265
266 /**
267 * @brief Test the pkt pool flags in efa_rdm_ep
268 *
269 * @param[in] state struct efa_resource that is managed by the framework
270 */
271 void test_efa_rdm_ep_pkt_pool_flags(struct efa_resource **state) {
272 struct efa_resource *resource = *state;
273
274 efa_env.huge_page_setting = EFA_ENV_HUGE_PAGE_DISABLED;
275 efa_unit_test_resource_construct(resource, FI_EP_RDM, EFA_FABRIC_NAME);
276 check_ep_pkt_pool_flags(resource->ep, OFI_BUFPOOL_NONSHARED);
277 }
278
279 /**
280 * @brief When the buf pool is created with OFI_BUFPOOL_NONSHARED,
281 * test if the allocated memory is page aligned.
282 *
283 * @param[in] state struct efa_resource that is managed by the framework
284 */
285 void test_efa_rdm_ep_pkt_pool_page_alignment(struct efa_resource **state)
286 {
287 int ret;
288 struct efa_rdm_pke *pkt_entry;
289 struct fid_ep *ep;
290 struct efa_rdm_ep *efa_rdm_ep;
291 struct efa_resource *resource = *state;
292
293 efa_unit_test_resource_construct(resource, FI_EP_RDM, EFA_FABRIC_NAME);
294
295 efa_env.huge_page_setting = EFA_ENV_HUGE_PAGE_DISABLED;
296 ret = fi_endpoint(resource->domain, resource->info, &ep, NULL);
297 assert_int_equal(ret, 0);
298 efa_rdm_ep = container_of(ep, struct efa_rdm_ep, base_ep.util_ep.ep_fid);
299 assert_int_equal(efa_rdm_ep->efa_rx_pkt_pool->attr.flags, OFI_BUFPOOL_NONSHARED);
300
301 pkt_entry = efa_rdm_pke_alloc(efa_rdm_ep, efa_rdm_ep->efa_rx_pkt_pool, EFA_RDM_PKE_FROM_EFA_RX_POOL);
302 assert_non_null(pkt_entry);
303 assert_true(((uintptr_t)ofi_buf_region(pkt_entry)->alloc_region % ofi_get_page_size()) == 0);
304 efa_rdm_pke_release_rx(pkt_entry);
305
306 fi_close(&ep->fid);
307 }
308
309 /**
310 * @brief When using LL128 protocol, test the packet allocated from read_copy_pkt_pool
311 * is 128 byte aligned.
312 *
313 * @param[in] state struct efa_resource that is managed by the framework
314 */
315 void test_efa_rdm_read_copy_pkt_pool_128_alignment(struct efa_resource **state)
316 {
317 int ret;
318 struct efa_rdm_pke *pkt_entry;
319 struct fid_ep *ep;
320 struct efa_rdm_ep *efa_rdm_ep;
321 struct efa_resource *resource = *state;
322 struct efa_domain *efa_domain = NULL;
323
324 efa_unit_test_resource_construct(resource, FI_EP_RDM, EFA_FABRIC_NAME);
325
326 /* rx_readcopy_pkt_pool is only created when application requested FI_HMEM */
327 efa_domain = container_of(resource->domain, struct efa_domain,
328 util_domain.domain_fid);
329 efa_domain->util_domain.mr_mode |= FI_MR_HMEM;
330
331 ret = fi_endpoint(resource->domain, resource->info, &ep, NULL);
332 assert_int_equal(ret, 0);
333 efa_rdm_ep = container_of(ep, struct efa_rdm_ep, base_ep.util_ep.ep_fid);
334 efa_rdm_ep->sendrecv_in_order_aligned_128_bytes = 1;
335
336 pkt_entry =
337 efa_rdm_pke_alloc(efa_rdm_ep, efa_rdm_ep->rx_readcopy_pkt_pool,
338 EFA_RDM_PKE_FROM_READ_COPY_POOL);
339 assert_non_null(pkt_entry);
340 efa_rdm_ep->rx_readcopy_pkt_pool_used++;
341 assert(ofi_is_addr_aligned((void *) pkt_entry->wiredata,
342 EFA_RDM_IN_ORDER_ALIGNMENT));
343 efa_rdm_pke_release_rx(pkt_entry);
344
345 fi_close(&ep->fid);
346 }
347
348 /**
349 * @brief When using LL128 protocol, the copy method is local read.
350 *
351 * @param[in] state struct efa_resource that is managed by the framework
352 */
353 void test_efa_rdm_pke_get_available_copy_methods_align128(struct efa_resource **state)
354 {
355 int ret;
356 struct efa_rdm_ep *efa_rdm_ep;
357 struct efa_mr efa_mr;
358 struct efa_resource *resource = *state;
359 bool local_read_available, gdrcopy_available, cuda_memcpy_available;
360
361 efa_unit_test_resource_construct(resource, FI_EP_RDM, EFA_FABRIC_NAME);
362 efa_mr.peer.iface = FI_HMEM_CUDA;
363
364 efa_rdm_ep = container_of(resource->ep, struct efa_rdm_ep, base_ep.util_ep.ep_fid);
365 efa_rdm_ep->sendrecv_in_order_aligned_128_bytes = 1;
366
367 /* p2p is available */
368 g_efa_hmem_info[FI_HMEM_CUDA].p2p_supported_by_device = true;
369 efa_rdm_ep->hmem_p2p_opt = FI_HMEM_P2P_ENABLED;
370
371 /* RDMA read is supported */
372 efa_rdm_ep->use_device_rdma = true;
373 uint64_t caps = efa_rdm_ep_domain(efa_rdm_ep)->device->device_caps;
374 efa_rdm_ep_domain(efa_rdm_ep)->device->device_caps |=
375 EFADV_DEVICE_ATTR_CAPS_RDMA_READ;
376
377 ret = efa_rdm_pke_get_available_copy_methods(
378 efa_rdm_ep, &efa_mr, &local_read_available,
379 &cuda_memcpy_available, &gdrcopy_available);
380
381 efa_rdm_ep_domain(efa_rdm_ep)->device->device_caps = caps;
382
383 assert_int_equal(ret, 0);
384 assert_true(local_read_available);
385 assert_false(cuda_memcpy_available);
386 assert_false(gdrcopy_available);
387 }
388
389 /**
390 * @brief when delivery complete atomic was used and handshake packet has not been received
391 * verify the txe is queued
392 *
393 * @param[in] state struct efa_resource that is managed by the framework
394 */
395 void test_efa_rdm_ep_dc_atomic_queue_before_handshake(struct efa_resource **state)
396 {
397 struct efa_rdm_ep *efa_rdm_ep;
398 struct efa_rdm_peer *peer;
399 struct fi_ioc ioc = {0};
400 struct fi_rma_ioc rma_ioc = {0};
401 struct fi_msg_atomic msg = {0};
402 struct efa_resource *resource = *state;
403 struct efa_ep_addr raw_addr = {0};
404 size_t raw_addr_len = sizeof(struct efa_ep_addr);
405 fi_addr_t peer_addr;
406 int buf[1] = {0}, err, numaddr;
407 struct efa_rdm_ope *txe;
408
409 /* disable shm to force using efa device to send */
410 efa_unit_test_resource_construct_rdm_shm_disabled(resource);
411
412 /* create a fake peer */
413 err = fi_getname(&resource->ep->fid, &raw_addr, &raw_addr_len);
414 assert_int_equal(err, 0);
415 raw_addr.qpn = 1;
416 raw_addr.qkey = 0x1234;
417 numaddr = fi_av_insert(resource->av, &raw_addr, 1, &peer_addr, 0, NULL);
418 assert_int_equal(numaddr, 1);
419
420 msg.addr = peer_addr;
421
422 ioc.addr = buf;
423 ioc.count = 1;
424 msg.msg_iov = &ioc;
425 msg.iov_count = 1;
426
427 msg.rma_iov = &rma_ioc;
428 msg.rma_iov_count = 1;
429 msg.datatype = FI_INT32;
430 msg.op = FI_SUM;
431
432 efa_rdm_ep = container_of(resource->ep, struct efa_rdm_ep, base_ep.util_ep.ep_fid);
433
434 /* set peer->flag to EFA_RDM_PEER_REQ_SENT will make efa_rdm_atomic() think
435 * a REQ packet has been sent to the peer (so no need to send again)
436 * handshake has not been received, so we do not know whether the peer support DC
437 */
438 peer = efa_rdm_ep_get_peer(efa_rdm_ep, peer_addr);
439 peer->flags = EFA_RDM_PEER_REQ_SENT;
440 peer->is_local = false;
441
442 assert_true(dlist_empty(&efa_rdm_ep->txe_list));
443 err = fi_atomicmsg(resource->ep, &msg, FI_DELIVERY_COMPLETE);
444 /* DC has been reuquested, but ep do not know whether peer supports it, therefore
445 * the ope has been queued to domain->ope_queued_list
446 */
447 assert_int_equal(err, 0);
448 assert_int_equal(efa_unit_test_get_dlist_length(&efa_rdm_ep->txe_list), 1);
449 assert_int_equal(efa_unit_test_get_dlist_length(&(efa_rdm_ep_domain(efa_rdm_ep)->ope_queued_list)), 1);
450 txe = container_of(efa_rdm_ep_domain(efa_rdm_ep)->ope_queued_list.next, struct efa_rdm_ope, queued_entry);
451 assert_true((txe->op == ofi_op_atomic));
452 assert_true(txe->internal_flags & EFA_RDM_OPE_QUEUED_BEFORE_HANDSHAKE);
453 }
454
455 /**
456 * @brief when delivery complete send was used and handshake packet has not been received
457 * verify the txe is queued
458 *
459 * @param[in] state struct efa_resource that is managed by the framework
460 */
461 void test_efa_rdm_ep_dc_send_queue_before_handshake(struct efa_resource **state)
462 {
463 struct efa_rdm_ep *efa_rdm_ep;
464 struct efa_rdm_peer *peer;
465 struct fi_msg msg = {0};
466 struct iovec iov;
467 struct efa_resource *resource = *state;
468 struct efa_ep_addr raw_addr = {0};
469 size_t raw_addr_len = sizeof(struct efa_ep_addr);
470 fi_addr_t peer_addr;
471 int err, numaddr;
472 struct efa_rdm_ope *txe;
473
474 /* disable shm to force using efa device to send */
475 efa_unit_test_resource_construct_rdm_shm_disabled(resource);
476
477 /* create a fake peer */
478 err = fi_getname(&resource->ep->fid, &raw_addr, &raw_addr_len);
479 assert_int_equal(err, 0);
480 raw_addr.qpn = 1;
481 raw_addr.qkey = 0x1234;
482 numaddr = fi_av_insert(resource->av, &raw_addr, 1, &peer_addr, 0, NULL);
483 assert_int_equal(numaddr, 1);
484
485 msg.addr = peer_addr;
486 msg.iov_count = 1;
487 iov.iov_base = NULL;
488 iov.iov_len = 0;
489 msg.msg_iov = &iov;
490 msg.desc = NULL;
491
492 efa_rdm_ep = container_of(resource->ep, struct efa_rdm_ep, base_ep.util_ep.ep_fid);
493
494 /* set peer->flag to EFA_RDM_PEER_REQ_SENT will make efa_rdm_atomic() think
495 * a REQ packet has been sent to the peer (so no need to send again)
496 * handshake has not been received, so we do not know whether the peer support DC
497 */
498 peer = efa_rdm_ep_get_peer(efa_rdm_ep, peer_addr);
499 peer->flags = EFA_RDM_PEER_REQ_SENT;
500 peer->is_local = false;
501
502 assert_true(dlist_empty(&efa_rdm_ep->txe_list));
503 err = fi_sendmsg(resource->ep, &msg, FI_DELIVERY_COMPLETE);
504 /* DC has been reuquested, but ep do not know whether peer supports it, therefore
505 * the ope has been queued to domain->ope_queued_list
506 */
507 assert_int_equal(err, 0);
508 assert_int_equal(efa_unit_test_get_dlist_length(&efa_rdm_ep->txe_list), 1);
509 assert_int_equal(efa_unit_test_get_dlist_length(&(efa_rdm_ep_domain(efa_rdm_ep)->ope_queued_list)), 1);
510 txe = container_of(efa_rdm_ep_domain(efa_rdm_ep)->ope_queued_list.next, struct efa_rdm_ope, queued_entry);
511 assert_true((txe->op == ofi_op_msg));
512 assert_true(txe->internal_flags & EFA_RDM_OPE_QUEUED_BEFORE_HANDSHAKE);
513 }
514
515 /**
516 * @brief when delivery complete send was used and handshake packet has not been received
517 * verify the txes are queued before the number of requests reach EFA_RDM_MAX_QUEUED_OPE_BEFORE_HANDSHAKE.
518 * After reaching the limit, fi_send should return -FI_EAGAIN
519 *
520 * @param[in] state struct efa_resource that is managed by the framework
521 */
522 void test_efa_rdm_ep_dc_send_queue_limit_before_handshake(struct efa_resource **state)
523 {
524 struct efa_rdm_ep *efa_rdm_ep;
525 struct efa_rdm_peer *peer;
526 struct fi_msg msg = {0};
527 struct iovec iov;
528 struct efa_resource *resource = *state;
529 struct efa_ep_addr raw_addr = {0};
530 size_t raw_addr_len = sizeof(struct efa_ep_addr);
531 fi_addr_t peer_addr;
532 int err, numaddr;
533 int i;
534
535 /* disable shm to force using efa device to send */
536 efa_unit_test_resource_construct_rdm_shm_disabled(resource);
537
538 /* create a fake peer */
539 err = fi_getname(&resource->ep->fid, &raw_addr, &raw_addr_len);
540 assert_int_equal(err, 0);
541 raw_addr.qpn = 1;
542 raw_addr.qkey = 0x1234;
543 numaddr = fi_av_insert(resource->av, &raw_addr, 1, &peer_addr, 0, NULL);
544 assert_int_equal(numaddr, 1);
545
546 msg.addr = peer_addr;
547 msg.iov_count = 1;
548 iov.iov_base = NULL;
549 iov.iov_len = 0;
550 msg.msg_iov = &iov;
551 msg.desc = NULL;
552
553 efa_rdm_ep = container_of(resource->ep, struct efa_rdm_ep, base_ep.util_ep.ep_fid);
554
555 /* set peer->flag to EFA_RDM_PEER_REQ_SENT will make efa_rdm_atomic() think
556 * a REQ packet has been sent to the peer (so no need to send again)
557 * handshake has not been received, so we do not know whether the peer support DC
558 */
559 peer = efa_rdm_ep_get_peer(efa_rdm_ep, peer_addr);
560 peer->flags = EFA_RDM_PEER_REQ_SENT;
561 peer->is_local = false;
562
563 assert_true(dlist_empty(&efa_rdm_ep->txe_list));
564
565 for (i = 0; i < EFA_RDM_MAX_QUEUED_OPE_BEFORE_HANDSHAKE; i++) {
566 err = fi_sendmsg(resource->ep, &msg, FI_DELIVERY_COMPLETE);
567 assert_int_equal(err, 0);
568 }
569
570 assert_true(efa_rdm_ep->ope_queued_before_handshake_cnt == EFA_RDM_MAX_QUEUED_OPE_BEFORE_HANDSHAKE);
571 err = fi_sendmsg(resource->ep, &msg, FI_DELIVERY_COMPLETE);
572 assert_int_equal(err, -FI_EAGAIN);
573 }
574
575 /**
576 * @brief verify tx entry is queued for rma (read or write) request before handshake is made.
577 *
578 * @param[in] state struct efa_resource that is managed by the framework
579 * @param[in] op op code
580 */
581 void test_efa_rdm_ep_rma_queue_before_handshake(struct efa_resource **state, int op)
582 {
583 struct efa_resource *resource = *state;
584 struct efa_rdm_ep *efa_rdm_ep;
585 struct efa_ep_addr raw_addr = {0};
586 size_t raw_addr_len = sizeof(struct efa_ep_addr);
587 fi_addr_t peer_addr;
588 int num_addr;
589 const int buf_len = 8;
590 char buf[8] = {0};
591 int err;
592 uint64_t rma_addr, rma_key;
593 struct efa_rdm_ope *txe;
594 struct efa_rdm_peer *peer;
595
596 resource->hints = efa_unit_test_alloc_hints(FI_EP_RDM, EFA_FABRIC_NAME);
597 resource->hints->caps |= FI_MSG | FI_TAGGED | FI_RMA;
598 resource->hints->domain_attr->mr_mode |= MR_MODE_BITS;
599 efa_unit_test_resource_construct_with_hints(resource, FI_EP_RDM, FI_VERSION(1, 14),
600 resource->hints, true, true);
601
602 /* ensure we don't have RMA capability. */
603 efa_rdm_ep = container_of(resource->ep, struct efa_rdm_ep, base_ep.util_ep.ep_fid);
604
605 /* create a fake peer */
606 err = fi_getname(&resource->ep->fid, &raw_addr, &raw_addr_len);
607 assert_int_equal(err, 0);
608 raw_addr.qpn = 1;
609 raw_addr.qkey = 0x1234;
610 num_addr = fi_av_insert(resource->av, &raw_addr, 1, &peer_addr, 0, NULL);
611 assert_int_equal(num_addr, 1);
612
613 /* create a fake rma_key and address. fi_read should return before
614 * they are needed. */
615 rma_key = 0x1234;
616 rma_addr = (uint64_t) &buf;
617
618 /* set peer->flag to EFA_RDM_PEER_REQ_SENT will make efa_rdm_atomic() think
619 * a REQ packet has been sent to the peer (so no need to send again)
620 * handshake has not been received, so we do not know whether the peer support DC
621 */
622 peer = efa_rdm_ep_get_peer(efa_rdm_ep, peer_addr);
623 peer->flags = EFA_RDM_PEER_REQ_SENT;
624 peer->is_local = false;
625
626 assert_true(dlist_empty(&efa_rdm_ep->txe_list));
627
628 if (op == ofi_op_read_req) {
629 err = fi_read(resource->ep, buf, buf_len,
630 NULL, /* desc, not required */
631 peer_addr,
632 rma_addr,
633 rma_key,
634 NULL); /* context */
635 } else if (op == ofi_op_write) {
636 err = fi_write(resource->ep, buf, buf_len,
637 NULL, /* desc, not required */
638 peer_addr,
639 rma_addr,
640 rma_key,
641 NULL); /* context */
642 } else {
643 fprintf(stderr, "Unknown op code %d\n", op);
644 fail();
645 }
646 assert_int_equal(err, 0);
647 assert_int_equal(efa_unit_test_get_dlist_length(&efa_rdm_ep->txe_list), 1);
648 assert_int_equal(efa_unit_test_get_dlist_length(&(efa_rdm_ep_domain(efa_rdm_ep)->ope_queued_list)), 1);
649 txe = container_of(efa_rdm_ep_domain(efa_rdm_ep)->ope_queued_list.next, struct efa_rdm_ope, queued_entry);
650 assert_true((txe->op == op));
651 assert_true(txe->internal_flags & EFA_RDM_OPE_QUEUED_BEFORE_HANDSHAKE);
652 }
653
654 void test_efa_rdm_ep_write_queue_before_handshake(struct efa_resource **state)
655 {
656 test_efa_rdm_ep_rma_queue_before_handshake(state, ofi_op_write);
657 }
658
659 void test_efa_rdm_ep_read_queue_before_handshake(struct efa_resource **state)
660 {
661 test_efa_rdm_ep_rma_queue_before_handshake(state, ofi_op_read_req);
662 }
663
664 /**
665 * @brief Test the efa_rdm_ep_trigger_handshake function
666 * with different peer setup and check the txe flags
667 *
668 * @param state efa_resource
669 */
670 void test_efa_rdm_ep_trigger_handshake(struct efa_resource **state)
671 {
672 struct efa_rdm_ope *txe;
673 struct efa_rdm_ep *efa_rdm_ep;
674 struct efa_resource *resource = *state;
675 struct efa_rdm_peer *peer;
676 struct efa_ep_addr raw_addr = {0};
677 size_t raw_addr_len = sizeof(raw_addr);
678 fi_addr_t peer_addr = 0;
679
680 g_efa_unit_test_mocks.efa_rdm_ope_post_send = &efa_mock_efa_rdm_ope_post_send_return_mock;
681
682 efa_unit_test_resource_construct(resource, FI_EP_RDM, EFA_FABRIC_NAME);
683
684 efa_rdm_ep = container_of(resource->ep, struct efa_rdm_ep, base_ep.util_ep.ep_fid);
685
686 will_return_always(efa_mock_efa_rdm_ope_post_send_return_mock, FI_SUCCESS);
687
688 /* Create and register a fake peer */
689 assert_int_equal(fi_getname(&resource->ep->fid, &raw_addr, &raw_addr_len), 0);
690 raw_addr.qpn = 0;
691 raw_addr.qkey = 0x1234;
692
693 assert_int_equal(fi_av_insert(resource->av, &raw_addr, 1, &peer_addr, 0, NULL), 1);
694
695 peer = efa_rdm_ep_get_peer(efa_rdm_ep, peer_addr);
696 assert_non_null(peer);
697
698 /* No txe should have been allocated yet */
699 assert_true(dlist_empty(&efa_rdm_ep->txe_list));
700
701 /*
702 * When the peer already has made , the function should be a no-op
703 * and no txe is allocated
704 */
705 peer->flags |= EFA_RDM_PEER_HANDSHAKE_RECEIVED | EFA_RDM_PEER_REQ_SENT;
706 assert_int_equal(efa_rdm_ep_trigger_handshake(efa_rdm_ep, peer), FI_SUCCESS);
707 assert_true(dlist_empty(&efa_rdm_ep->txe_list));
708
709 /*
710 * Reset the peer flags to 0, now we should expect a txe allocated
711 */
712 peer->flags = 0;
713 assert_int_equal(efa_rdm_ep_trigger_handshake(efa_rdm_ep, peer), FI_SUCCESS);
714 assert_int_equal(efa_unit_test_get_dlist_length(&efa_rdm_ep->txe_list), 1);
715
716 txe = container_of(efa_rdm_ep->txe_list.next, struct efa_rdm_ope, ep_entry);
717
718 assert_true(txe->fi_flags & EFA_RDM_TXE_NO_COMPLETION);
719 assert_true(txe->fi_flags & EFA_RDM_TXE_NO_COUNTER);
720 assert_true(txe->internal_flags & EFA_RDM_OPE_INTERNAL);
721
722 efa_rdm_txe_release(txe);
723 }
724
725 /**
726 * @brief When local support unsolicited write, but the peer doesn't, fi_writedata
727 * (use rdma-write with imm) should fail as FI_EINVAL
728 *
729 * @param state struct efa_resource that is managed by the framework
730 */
731 void test_efa_rdm_ep_rma_inconsistent_unsolicited_write_recv(struct efa_resource **state)
732 {
733 struct efa_resource *resource = *state;
734 struct efa_rdm_ep *efa_rdm_ep;
735 struct efa_ep_addr raw_addr = {0};
736 size_t raw_addr_len = sizeof(struct efa_ep_addr);
737 fi_addr_t peer_addr;
738 int num_addr;
739 const int buf_len = 8;
740 char buf[8] = {0};
741 int err;
742 uint64_t rma_addr, rma_key;
743 struct efa_rdm_peer *peer;
744
745 resource->hints = efa_unit_test_alloc_hints(FI_EP_RDM, EFA_FABRIC_NAME);
746 resource->hints->caps |= FI_MSG | FI_TAGGED | FI_RMA;
747 resource->hints->domain_attr->mr_mode |= MR_MODE_BITS;
748 efa_unit_test_resource_construct_with_hints(resource, FI_EP_RDM, FI_VERSION(1, 22),
749 resource->hints, true, true);
750
751 efa_rdm_ep = container_of(resource->ep, struct efa_rdm_ep, base_ep.util_ep.ep_fid);
752
753 /**
754 * TODO: It's better to mock this function
755 * so we can test on platform that doesn't
756 * support rdma-write.
757 */
758 if (!(efa_rdm_ep_support_rdma_write(efa_rdm_ep)))
759 skip();
760
761 /* Make local ep support unsolicited write recv */
762 efa_rdm_ep->extra_info[0] |= EFA_RDM_EXTRA_FEATURE_UNSOLICITED_WRITE_RECV;
763
764 /* create a fake peer */
765 err = fi_getname(&resource->ep->fid, &raw_addr, &raw_addr_len);
766 assert_int_equal(err, 0);
767 raw_addr.qpn = 1;
768 raw_addr.qkey = 0x1234;
769 num_addr = fi_av_insert(resource->av, &raw_addr, 1, &peer_addr, 0, NULL);
770 assert_int_equal(num_addr, 1);
771
772 /* create a fake rma_key and address. fi_read should return before
773 * they are needed. */
774 rma_key = 0x1234;
775 rma_addr = (uint64_t) &buf;
776
777 /*
778 * Fake a peer that has made handshake and
779 * does not support unsolicited write recv
780 */
781 peer = efa_rdm_ep_get_peer(efa_rdm_ep, peer_addr);
782 peer->flags |= EFA_RDM_PEER_HANDSHAKE_RECEIVED;
783 peer->extra_info[0] |= EFA_RDM_EXTRA_FEATURE_RDMA_WRITE;
784 peer->extra_info[0] &= ~EFA_RDM_EXTRA_FEATURE_UNSOLICITED_WRITE_RECV;
785 /* make sure shm is not used */
786 peer->is_local = false;
787
788 err = fi_writedata(resource->ep, buf, buf_len,
789 NULL, /* desc, not required */
790 0x1234,
791 peer_addr,
792 rma_addr,
793 rma_key,
794 NULL); /* context */
795 assert_int_equal(err, -FI_EOPNOTSUPP);
796 }
797
798 /**
799 * @brief verify that when shm was used to send a small message (<4k), no copy was performed.
800 *
801 * @param[in] state struct efa_resource that is managed by the framework
802 */
803 void test_efa_rdm_ep_send_with_shm_no_copy(struct efa_resource **state)
804 {
805 struct efa_resource *resource = *state;
806 struct efa_ep_addr raw_addr = {0};
807 size_t raw_addr_len = sizeof(struct efa_ep_addr);
808 fi_addr_t peer_addr;
809 int num_addr;
810 int buff_len = 8;
811 char buff[8] = {0};
812 int err;
813
814 efa_unit_test_resource_construct(resource, FI_EP_RDM, EFA_FABRIC_NAME);
815
816 /* create a fake peer */
817 err = fi_getname(&resource->ep->fid, &raw_addr, &raw_addr_len);
818 assert_int_equal(err, 0);
819 raw_addr.qpn = 1;
820 raw_addr.qkey = 0x1234;
821 num_addr = fi_av_insert(resource->av, &raw_addr, 1, &peer_addr, 0, NULL);
822 assert_int_equal(num_addr, 1);
823
824 g_ofi_copy_from_hmem_iov_call_counter = 0;
825 g_efa_unit_test_mocks.ofi_copy_from_hmem_iov = efa_mock_ofi_copy_from_hmem_iov_inc_counter;
826
827 err = fi_send(resource->ep, buff, buff_len,
828 NULL /* desc, which is not required by shm */,
829 peer_addr,
830 NULL /* context */);
831
832 assert_int_equal(g_ofi_copy_from_hmem_iov_call_counter, 0);
833 }
834
835 /**
836 * @brief verify error is generated for RMA on non-RMA-enabled EP.
837 *
838 * @param[in] state struct efa_resource that is managed by the framework
839 */
840 void test_efa_rdm_ep_rma_without_caps(struct efa_resource **state)
841 {
842 struct efa_resource *resource = *state;
843 struct efa_rdm_ep *efa_rdm_ep;
844 struct efa_ep_addr raw_addr = {0};
845 size_t raw_addr_len = sizeof(struct efa_ep_addr);
846 fi_addr_t peer_addr;
847 int num_addr;
848 const int buf_len = 8;
849 char buf[8] = {0};
850 int err;
851 uint64_t rma_addr, rma_key;
852
853 resource->hints = efa_unit_test_alloc_hints(FI_EP_RDM, EFA_FABRIC_NAME);
854 resource->hints->caps |= FI_MSG | FI_TAGGED;
855 resource->hints->caps &= ~FI_RMA;
856 resource->hints->domain_attr->mr_mode |= MR_MODE_BITS;
857 efa_unit_test_resource_construct_with_hints(resource, FI_EP_RDM, FI_VERSION(1, 14),
858 resource->hints, true, true);
859
860 /* ensure we don't have RMA capability. */
861 efa_rdm_ep = container_of(resource->ep, struct efa_rdm_ep, base_ep.util_ep.ep_fid);
862 assert_int_equal( efa_rdm_ep->base_ep.info->caps & FI_RMA, 0);
863
864 /* create a fake peer */
865 err = fi_getname(&resource->ep->fid, &raw_addr, &raw_addr_len);
866 assert_int_equal(err, 0);
867 raw_addr.qpn = 1;
868 raw_addr.qkey = 0x1234;
869 num_addr = fi_av_insert(resource->av, &raw_addr, 1, &peer_addr, 0, NULL);
870 assert_int_equal(num_addr, 1);
871
872 /* create a fake rma_key and address. fi_read should return before
873 * they are needed. */
874 rma_key = 0x1234;
875 rma_addr = (uint64_t) &buf;
876 err = fi_read(resource->ep, buf, buf_len,
877 NULL, /* desc, not required */
878 peer_addr,
879 rma_addr,
880 rma_key,
881 NULL); /* context */
882
883 assert_int_equal(err, -FI_EOPNOTSUPP);
884 }
885
886 /**
887 * @brief verify error is generated for Atomic operations on non-Atomic-enabled EP.
888 *
889 * @param[in] state struct efa_resource that is managed by the framework
890 */
891 void test_efa_rdm_ep_atomic_without_caps(struct efa_resource **state)
892 {
893 struct efa_resource *resource = *state;
894 struct efa_rdm_ep *efa_rdm_ep;
895 struct efa_ep_addr raw_addr = {0};
896 size_t raw_addr_len = sizeof(struct efa_ep_addr);
897 fi_addr_t peer_addr;
898 int num_addr;
899 const int buf_len = 8;
900 char buf[8] = {0};
901 int err;
902 uint64_t rma_addr, rma_key;
903
904 resource->hints = efa_unit_test_alloc_hints(FI_EP_RDM, EFA_FABRIC_NAME);
905 resource->hints->caps |= FI_MSG | FI_TAGGED;
906 resource->hints->caps &= ~FI_ATOMIC;
907 resource->hints->domain_attr->mr_mode |= MR_MODE_BITS;
908 efa_unit_test_resource_construct_with_hints(resource, FI_EP_RDM, FI_VERSION(1, 14),
909 resource->hints, true, true);
910
911 /* ensure we don't have ATOMIC capability. */
912 efa_rdm_ep = container_of(resource->ep, struct efa_rdm_ep, base_ep.util_ep.ep_fid);
913 assert_int_equal( efa_rdm_ep->base_ep.info->caps & FI_ATOMIC, 0);
914
915 /* create a fake peer */
916 err = fi_getname(&resource->ep->fid, &raw_addr, &raw_addr_len);
917 assert_int_equal(err, 0);
918 raw_addr.qpn = 1;
919 raw_addr.qkey = 0x1234;
920 num_addr = fi_av_insert(resource->av, &raw_addr, 1, &peer_addr, 0, NULL);
921 assert_int_equal(num_addr, 1);
922
923 /* create a fake rma_key and address. fi_atomic should return before
924 * they are needed. */
925 rma_key = 0x1234;
926 rma_addr = (uint64_t) &buf;
927 err = fi_atomic(resource->ep, buf, buf_len,
928 NULL, /* desc, not required */
929 peer_addr,
930 rma_addr,
931 rma_key,
932 FI_INT32,
933 FI_SUM,
934 NULL); /* context */
935
936 assert_int_equal(err, -FI_EOPNOTSUPP);
937 }
938
939 /*
940 * Check fi_getopt return with different input opt_len
941 */
942 void test_efa_rdm_ep_getopt(struct efa_resource **state, size_t opt_len, int expected_return)
943 {
944 struct efa_resource *resource = *state;
945 size_t opt_val;
946 size_t opt_len_temp;
947 size_t i;
948 int ret;
949 int opt_names[] = {
950 FI_OPT_MIN_MULTI_RECV,
951 FI_OPT_EFA_RNR_RETRY,
952 FI_OPT_FI_HMEM_P2P,
953 FI_OPT_EFA_EMULATED_READ,
954 FI_OPT_EFA_EMULATED_WRITE,
955 FI_OPT_EFA_EMULATED_ATOMICS,
956 };
957 size_t num_opt_names = sizeof(opt_names) / sizeof(int);
958
959 efa_unit_test_resource_construct(resource, FI_EP_RDM, EFA_FABRIC_NAME);
960
961 for (i = 0; i < num_opt_names; i++) {
962 opt_len_temp = opt_len;
963 ret = fi_getopt(&resource->ep->fid, FI_OPT_ENDPOINT, opt_names[i], &opt_val, &opt_len_temp);
964 assert_int_equal(ret, expected_return);
965 }
966 }
967
968 /* undersized optlen should return -FI_ETOOSMALL */
969 void test_efa_rdm_ep_getopt_undersized_optlen(struct efa_resource **state)
970 {
971 test_efa_rdm_ep_getopt(state, 0, -FI_ETOOSMALL);
972 }
973
974 /* oversized optlen should return FI_SUCCESS */
975 void test_efa_rdm_ep_getopt_oversized_optlen(struct efa_resource **state)
976 {
977 test_efa_rdm_ep_getopt(state, 16, FI_SUCCESS);
978 }
979
980 void test_efa_rdm_ep_setopt_shared_memory_permitted(struct efa_resource **state)
981 {
982 struct efa_resource *resource = *state;
983 struct efa_rdm_ep *ep;
984
985 /* disable shm to force using efa device to send */
986 efa_unit_test_resource_construct_rdm_shm_disabled(resource);
987
988 ep = container_of(resource->ep, struct efa_rdm_ep,
989 base_ep.util_ep.ep_fid);
990
991 assert_null(ep->shm_ep);
992 }
993
994 /**
995 * @brief Test fi_enable with different optval of fi_setopt for
996 * FI_OPT_EFA_WRITE_IN_ORDER_ALIGNED_128_BYTES optname.
997 * @param state struct efa_resource that is managed by the framework
998 * @param expected_status expected return status of fi_enable
999 * @param optval the optval passed to fi_setopt
1000 */
1001 void test_efa_rdm_ep_enable_qp_in_order_aligned_128_bytes_common(struct efa_resource **state, int expected_status, bool optval)
1002 {
1003 struct efa_resource *resource = *state;
1004
1005 efa_unit_test_resource_construct_ep_not_enabled(resource, FI_EP_RDM, EFA_FABRIC_NAME);
1006
1007 /* fi_setopt should always succeed */
1008 assert_int_equal(fi_setopt(&resource->ep->fid, FI_OPT_ENDPOINT,
1009 FI_OPT_EFA_WRITE_IN_ORDER_ALIGNED_128_BYTES, &optval,
1010 sizeof(optval)), expected_status);
1011 }
1012
1013 #if HAVE_EFA_DATA_IN_ORDER_ALIGNED_128_BYTES
1014 /**
1015 * @brief Test the case where fi_enable should return success
1016 *
1017 * @param state struct efa_resource that is managed by the framework
1018 */
1019 void test_efa_rdm_ep_enable_qp_in_order_aligned_128_bytes_good(struct efa_resource **state)
1020 {
1021 /* mock ibv_query_qp_data_in_order to return required capability */
1022 g_efa_unit_test_mocks.ibv_query_qp_data_in_order = &efa_mock_ibv_query_qp_data_in_order_return_in_order_aligned_128_bytes;
1023 test_efa_rdm_ep_enable_qp_in_order_aligned_128_bytes_common(state, FI_SUCCESS, true);
1024 }
1025
1026 /**
1027 * @brief Test the case where fi_enable should return -FI_EOPNOTSUPP
1028 *
1029 * @param state struct efa_resource that is managed by the framework
1030 */
1031 void test_efa_rdm_ep_enable_qp_in_order_aligned_128_bytes_bad(struct efa_resource **state)
1032 {
1033 /* mock ibv_query_qp_data_in_order to return zero capability */
1034 g_efa_unit_test_mocks.ibv_query_qp_data_in_order = &efa_mock_ibv_query_qp_data_in_order_return_0;
1035 test_efa_rdm_ep_enable_qp_in_order_aligned_128_bytes_common(state, -FI_EOPNOTSUPP, true);
1036 }
1037
1038 #else
1039
1040 void test_efa_rdm_ep_enable_qp_in_order_aligned_128_bytes_good(struct efa_resource **state)
1041 {
1042 test_efa_rdm_ep_enable_qp_in_order_aligned_128_bytes_common(state, FI_SUCCESS, false);
1043 }
1044
1045 void test_efa_rdm_ep_enable_qp_in_order_aligned_128_bytes_bad(struct efa_resource **state)
1046 {
1047 test_efa_rdm_ep_enable_qp_in_order_aligned_128_bytes_common(state, -FI_EOPNOTSUPP, true);
1048 }
1049
1050 #endif
1051
1052 static void test_efa_rdm_ep_use_zcpy_rx_impl(struct efa_resource *resource,
1053 bool cuda_p2p_disabled,
1054 bool cuda_p2p_supported,
1055 bool expected_use_zcpy_rx)
1056 {
1057 struct efa_rdm_ep *ep;
1058 size_t max_msg_size = 1000;
1059 size_t inject_msg_size = 0;
1060 size_t inject_rma_size = 0;
1061 bool shm_permitted = false;
1062 ofi_hmem_disable_p2p = cuda_p2p_disabled;
1063
1064 efa_unit_test_resource_construct_with_hints(resource, FI_EP_RDM, FI_VERSION(1, 14),
1065 resource->hints, false, true);
1066
1067 /* System memory P2P should always be enabled */
1068 assert_true(g_efa_hmem_info[FI_HMEM_SYSTEM].initialized);
1069 assert_true(g_efa_hmem_info[FI_HMEM_SYSTEM].p2p_supported_by_device);
1070
1071 /**
1072 * We want to be able to run this test on any platform:
1073 * 1. Fake CUDA support.
1074 * 2. Disable all other hmem ifaces.
1075 */
1076 g_efa_hmem_info[FI_HMEM_CUDA].initialized = true;
1077 g_efa_hmem_info[FI_HMEM_CUDA].p2p_supported_by_device = cuda_p2p_supported;
1078
1079 g_efa_hmem_info[FI_HMEM_NEURON].initialized = false;
1080 g_efa_hmem_info[FI_HMEM_SYNAPSEAI].initialized = false;
1081
1082 ep = container_of(resource->ep, struct efa_rdm_ep, base_ep.util_ep.ep_fid);
1083
1084 if (cuda_p2p_supported)
1085 ep->hmem_p2p_opt = FI_HMEM_P2P_ENABLED;
1086
1087 /* Set sufficiently small max_msg_size */
1088 assert_int_equal(fi_setopt(&resource->ep->fid, FI_OPT_ENDPOINT, FI_OPT_MAX_MSG_SIZE,
1089 &max_msg_size, sizeof max_msg_size), 0);
1090
1091 /* Disable shm */
1092 assert_int_equal(fi_setopt(&resource->ep->fid, FI_OPT_ENDPOINT, FI_OPT_SHARED_MEMORY_PERMITTED,
1093 &shm_permitted, sizeof shm_permitted), 0);
1094
1095 assert_true(ep->base_ep.max_msg_size == max_msg_size);
1096
1097 /* Enable EP */
1098 assert_int_equal(fi_enable(resource->ep), 0);
1099
1100 assert_true(ep->use_zcpy_rx == expected_use_zcpy_rx);
1101
1102 assert_int_equal(fi_getopt(&resource->ep->fid, FI_OPT_ENDPOINT, FI_OPT_INJECT_MSG_SIZE,
1103 &inject_msg_size, &(size_t){sizeof inject_msg_size}), 0);
1104 assert_int_equal(ep->base_ep.inject_msg_size, inject_msg_size);
1105
1106 assert_int_equal(fi_getopt(&resource->ep->fid, FI_OPT_ENDPOINT, FI_OPT_INJECT_RMA_SIZE,
1107 &inject_rma_size, &(size_t){sizeof inject_rma_size}), 0);
1108 assert_int_equal(ep->base_ep.inject_rma_size, inject_rma_size);
1109
1110 if (expected_use_zcpy_rx) {
1111 assert_int_equal(inject_msg_size, efa_rdm_ep_domain(ep)->device->efa_attr.inline_buf_size);
1112 assert_int_equal(inject_rma_size, efa_rdm_ep_domain(ep)->device->efa_attr.inline_buf_size);
1113 } else {
1114 assert_int_equal(inject_msg_size, resource->info->tx_attr->inject_size);
1115 assert_int_equal(inject_rma_size, resource->info->tx_attr->inject_size);
1116 }
1117 /* restore global variable */
1118 ofi_hmem_disable_p2p = 0;
1119 }
1120
1121 /**
1122 * @brief Verify zcpy_rx is enabled when the following requirements are met:
1123 * 1. app doesn't require FI_ORDER_SAS in tx or rx's msg_order
1124 * 2. app uses FI_MSG_PREFIX mode
1125 * 3. app's max msg size is smaller than mtu_size - prefix_size
1126 * 4. app doesn't use FI_DIRECTED_RECV, FI_TAGGED, FI_ATOMIC capability
1127 */
1128 void test_efa_rdm_ep_user_zcpy_rx_disabled(struct efa_resource **state)
1129 {
1130 struct efa_resource *resource = *state;
1131
1132 resource->hints = efa_unit_test_alloc_hints(FI_EP_RDM, EFA_FABRIC_NAME);
1133 assert_non_null(resource->hints);
1134
1135 resource->hints->mode = FI_MSG_PREFIX;
1136 resource->hints->caps = FI_MSG;
1137
1138 test_efa_rdm_ep_use_zcpy_rx_impl(resource, false, true, true);
1139 }
1140
1141 /**
1142 * @brief Verify zcpy_rx is disabled if CUDA P2P is explictly disabled
1143 */
1144 void test_efa_rdm_ep_user_disable_p2p_zcpy_rx_disabled(struct efa_resource **state)
1145 {
1146 struct efa_resource *resource = *state;
1147
1148 resource->hints = efa_unit_test_alloc_hints(FI_EP_RDM, EFA_FABRIC_NAME);
1149 assert_non_null(resource->hints);
1150
1151 resource->hints->mode = FI_MSG_PREFIX;
1152 resource->hints->caps = FI_MSG;
1153
1154 test_efa_rdm_ep_use_zcpy_rx_impl(resource, true, false, false);
1155 }
1156
1157 /**
1158 * @brief When sas is requested for either tx or rx. zcpy will be disabled
1159 */
1160 void test_efa_rdm_ep_user_zcpy_rx_unhappy_due_to_sas(struct efa_resource **state)
1161 {
1162 struct efa_resource *resource = *state;
1163
1164 resource->hints = efa_unit_test_alloc_hints(FI_EP_RDM, EFA_FABRIC_NAME);
1165 assert_non_null(resource->hints);
1166
1167 resource->hints->tx_attr->msg_order = FI_ORDER_SAS;
1168 resource->hints->rx_attr->msg_order = FI_ORDER_SAS;
1169 resource->hints->mode = FI_MSG_PREFIX;
1170 resource->hints->caps = FI_MSG;
1171
1172 test_efa_rdm_ep_use_zcpy_rx_impl(resource, false, true, false);
1173 }
1174
1175 /**
1176 * @brief Verify zcpy_rx is disabled if CUDA P2P is enabled but not supported
1177 */
1178 void test_efa_rdm_ep_user_p2p_not_supported_zcpy_rx_happy(struct efa_resource **state)
1179 {
1180 struct efa_resource *resource = *state;
1181
1182 resource->hints = efa_unit_test_alloc_hints(FI_EP_RDM, EFA_FABRIC_NAME);
1183 assert_non_null(resource->hints);
1184
1185 resource->hints->mode = FI_MSG_PREFIX;
1186 resource->hints->caps = FI_MSG;
1187
1188 test_efa_rdm_ep_use_zcpy_rx_impl(resource, false, false, false);
1189 }
1190
1191 /**
1192 * @brief Verify zcpy_rx is disabled if FI_MR_LOCAL is not set
1193 */
1194 void test_efa_rdm_ep_user_zcpy_rx_unhappy_due_to_no_mr_local(struct efa_resource **state)
1195 {
1196 struct efa_resource *resource = *state;
1197
1198 resource->hints = efa_unit_test_alloc_hints(FI_EP_RDM, EFA_FABRIC_NAME);
1199 assert_non_null(resource->hints);
1200
1201 resource->hints->caps = FI_MSG;
1202 resource->hints->domain_attr->mr_mode &= ~FI_MR_LOCAL;
1203
1204 test_efa_rdm_ep_use_zcpy_rx_impl(resource, false, true, false);
1205 }
1206
1207 void test_efa_rdm_ep_close_discard_posted_recv(struct efa_resource **state)
1208 {
1209 struct efa_resource *resource = *state;
1210 char buf[16];
1211
1212 efa_unit_test_resource_construct(resource, FI_EP_RDM, EFA_FABRIC_NAME);
1213
1214 /* Post recv and then close ep */
1215 assert_int_equal(fi_recv(resource->ep, (void *) buf, 16, NULL, FI_ADDR_UNSPEC, NULL), 0);
1216
1217 assert_int_equal(fi_close(&resource->ep->fid), 0);
1218
1219 /* CQ should be empty and no err entry */
1220 assert_int_equal(fi_cq_read(resource->cq, NULL, 1), -FI_EAGAIN);
1221
1222 /* Reset to NULL to avoid test reaper closing again */
1223 resource->ep = NULL;
1224 }
1225
1226 void test_efa_rdm_ep_zcpy_recv_cancel(struct efa_resource **state)
1227 {
1228 struct efa_resource *resource = *state;
1229 struct fi_context cancel_context = {0};
1230 struct efa_unit_test_buff recv_buff;
1231
1232 resource->hints = efa_unit_test_alloc_hints(FI_EP_RDM, EFA_FABRIC_NAME);
1233 assert_non_null(resource->hints);
1234
1235 resource->hints->caps = FI_MSG;
1236
1237 /* enable zero-copy recv mode in ep */
1238 test_efa_rdm_ep_use_zcpy_rx_impl(resource, false, true, true);
1239
1240 /* Construct a recv buffer with mr */
1241 efa_unit_test_buff_construct(&recv_buff, resource, 16);
1242
1243 assert_int_equal(fi_recv(resource->ep, recv_buff.buff, recv_buff.size, fi_mr_desc(recv_buff.mr), FI_ADDR_UNSPEC, &cancel_context), 0);
1244
1245 assert_int_equal(fi_cancel((struct fid *)resource->ep, &cancel_context), -FI_EOPNOTSUPP);
1246
1247 /**
1248 * the buf is still posted to rdma-core, so unregistering mr can
1249 * return non-zero. Currently ignore this failure.
1250 */
1251 (void) fi_close(&recv_buff.mr->fid);
1252 free(recv_buff.buff);
1253 }
1254
1255 /**
1256 * @brief When user posts more than rx size fi_recv, we should return eagain and make sure
1257 * there is no rx entry leaked
1258 */
1259 void test_efa_rdm_ep_zcpy_recv_eagain(struct efa_resource **state)
1260 {
1261 struct efa_resource *resource = *state;
1262 struct efa_unit_test_buff recv_buff;
1263 int i;
1264 struct efa_rdm_ep *efa_rdm_ep;
1265
1266 resource->hints = efa_unit_test_alloc_hints(FI_EP_RDM, EFA_FABRIC_NAME);
1267 assert_non_null(resource->hints);
1268
1269 resource->hints->caps = FI_MSG;
1270
1271 /* enable zero-copy recv mode in ep */
1272 test_efa_rdm_ep_use_zcpy_rx_impl(resource, false, true, true);
1273
1274 efa_rdm_ep = container_of(resource->ep, struct efa_rdm_ep, base_ep.util_ep.ep_fid);
1275
1276 /* Construct a recv buffer with mr */
1277 efa_unit_test_buff_construct(&recv_buff, resource, 16);
1278
1279 for (i = 0; i < efa_rdm_ep->base_ep.info->rx_attr->size; i++)
1280 assert_int_equal(fi_recv(resource->ep, recv_buff.buff, recv_buff.size, fi_mr_desc(recv_buff.mr), FI_ADDR_UNSPEC, NULL), 0);
1281
1282 /* we should have rx number of rx entry before and after the extra recv post */
1283 assert_true(efa_unit_test_get_dlist_length(&efa_rdm_ep->rxe_list) == efa_rdm_ep->base_ep.info->rx_attr->size);
1284 assert_int_equal(fi_recv(resource->ep, recv_buff.buff, recv_buff.size, fi_mr_desc(recv_buff.mr), FI_ADDR_UNSPEC, NULL), -FI_EAGAIN);
1285 assert_true(efa_unit_test_get_dlist_length(&efa_rdm_ep->rxe_list) == efa_rdm_ep->base_ep.info->rx_attr->size);
1286
1287 /**
1288 * the buf is still posted to rdma-core, so unregistering mr can
1289 * return non-zero. Currently ignore this failure.
1290 */
1291 (void) fi_close(&recv_buff.mr->fid);
1292 free(recv_buff.buff);
1293 }
1294
1295 /**
1296 * @brief when efa_rdm_ep_post_handshake_error failed due to pkt pool exhaustion,
1297 * make sure both txe is cleaned
1298 *
1299 * @param[in] state struct efa_resource that is managed by the framework
1300 */
1301 void test_efa_rdm_ep_post_handshake_error_handling_pke_exhaustion(struct efa_resource **state)
1302 {
1303 struct efa_rdm_ep *efa_rdm_ep;
1304 struct efa_rdm_peer *peer;
1305 struct efa_resource *resource = *state;
1306 struct efa_ep_addr raw_addr = {0};
1307 size_t raw_addr_len = sizeof(struct efa_ep_addr);
1308 fi_addr_t peer_addr;
1309 int err, numaddr;
1310 struct efa_rdm_pke **pkt_entry_vec;
1311 int i;
1312 size_t tx_size;
1313
1314 /* disable shm to force using efa device to send */
1315 efa_unit_test_resource_construct_rdm_shm_disabled(resource);
1316
1317 /* create a fake peer */
1318 err = fi_getname(&resource->ep->fid, &raw_addr, &raw_addr_len);
1319 assert_int_equal(err, 0);
1320 raw_addr.qpn = 1;
1321 raw_addr.qkey = 0x1234;
1322 numaddr = fi_av_insert(resource->av, &raw_addr, 1, &peer_addr, 0, NULL);
1323 assert_int_equal(numaddr, 1);
1324
1325 efa_rdm_ep = container_of(resource->ep, struct efa_rdm_ep, base_ep.util_ep.ep_fid);
1326 tx_size = efa_rdm_ep->base_ep.info->tx_attr->size;
1327
1328 /* set peer->flag to EFA_RDM_PEER_REQ_SENT will make efa_rdm_atomic() think
1329 * a REQ packet has been sent to the peer (so no need to send again)
1330 * handshake has not been received, so we do not know whether the peer support DC
1331 */
1332 peer = efa_rdm_ep_get_peer(efa_rdm_ep, peer_addr);
1333 peer->flags = EFA_RDM_PEER_REQ_SENT;
1334 peer->is_local = false;
1335
1336 pkt_entry_vec = calloc(tx_size, sizeof(struct efa_rdm_pke *));
1337 assert_non_null(pkt_entry_vec);
1338
1339 /* Exhaust the tx pkt pool */
1340 for (i = 0; i < tx_size; i++) {
1341 pkt_entry_vec[i] = efa_rdm_pke_alloc(efa_rdm_ep, efa_rdm_ep->efa_tx_pkt_pool, EFA_RDM_PKE_FROM_EFA_TX_POOL);
1342 assert_non_null(pkt_entry_vec[i]);
1343 }
1344
1345 /* txe list should be empty before and after the failed handshake post call */
1346 assert_true(dlist_empty(&efa_rdm_ep->txe_list));
1347 assert_int_equal(efa_rdm_ep_post_handshake(efa_rdm_ep, peer), -FI_EAGAIN);
1348 assert_true(dlist_empty(&efa_rdm_ep->txe_list));
1349
1350 for (i = 0; i < tx_size; i++)
1351 efa_rdm_pke_release_tx(pkt_entry_vec[i]);
1352
1353 free(pkt_entry_vec);
1354 }
1355
1356 static
1357 void test_efa_rdm_ep_rx_refill_impl(struct efa_resource **state, int threshold, int rx_size)
1358 {
1359 struct efa_resource *resource = *state;
1360 struct efa_rdm_ep *efa_rdm_ep;
1361 struct efa_rdm_pke *pkt_entry;
1362 int i;
1363 size_t threshold_orig;
1364
1365 if (threshold < 4 || rx_size < 4) {
1366 fprintf(stderr, "Too small threshold or rx_size for this test\n");
1367 fail();
1368 }
1369
1370 threshold_orig = efa_env.internal_rx_refill_threshold;
1371
1372 efa_env.internal_rx_refill_threshold = threshold;
1373
1374 resource->hints = efa_unit_test_alloc_hints(FI_EP_RDM, EFA_FABRIC_NAME);
1375 assert_non_null(resource->hints);
1376 resource->hints->rx_attr->size = rx_size;
1377 efa_unit_test_resource_construct_with_hints(resource, FI_EP_RDM, FI_VERSION(1, 14),
1378 resource->hints, true, true);
1379
1380 efa_rdm_ep = container_of(resource->ep, struct efa_rdm_ep, base_ep.util_ep.ep_fid);
1381 assert_int_equal(efa_rdm_ep_get_rx_pool_size(efa_rdm_ep), rx_size);
1382
1383 /* Grow the rx pool and post rx pkts */
1384 efa_rdm_ep_post_internal_rx_pkts(efa_rdm_ep);
1385 assert_int_equal(efa_rdm_ep->efa_rx_pkts_posted, efa_rdm_ep_get_rx_pool_size(efa_rdm_ep));
1386
1387 assert_int_equal(efa_rdm_ep->efa_rx_pkts_to_post, 0);
1388 for (i = 0; i < 4; i++) {
1389 pkt_entry = ofi_bufpool_get_ibuf(efa_rdm_ep->efa_rx_pkt_pool, i);
1390 assert_non_null(pkt_entry);
1391 efa_rdm_pke_release_rx(pkt_entry);
1392 }
1393 assert_int_equal(efa_rdm_ep->efa_rx_pkts_to_post, 4);
1394
1395 efa_rdm_ep_bulk_post_internal_rx_pkts(efa_rdm_ep);
1396
1397 /**
1398 * efa_rx_pkts_to_post < FI_EFA_RX_REFILL_THRESHOLD
1399 * pkts should NOT be refilled
1400 */
1401 assert_int_equal(efa_rdm_ep->efa_rx_pkts_to_post, 4);
1402 assert_int_equal(efa_rdm_ep->efa_rx_pkts_posted, rx_size);
1403
1404 /* releasing more pkts to reach the threshold or rx_size*/
1405 for (i = 4; i < MIN(rx_size, threshold); i++) {
1406 pkt_entry = ofi_bufpool_get_ibuf(efa_rdm_ep->efa_rx_pkt_pool, i);
1407 assert_non_null(pkt_entry);
1408 efa_rdm_pke_release_rx(pkt_entry);
1409 }
1410
1411 assert_int_equal(efa_rdm_ep->efa_rx_pkts_to_post, i);
1412
1413 efa_rdm_ep_bulk_post_internal_rx_pkts(efa_rdm_ep);
1414
1415 /**
1416 * efa_rx_pkts_to_post == min(FI_EFA_RX_REFILL_THRESHOLD, FI_EFA_RX_SIZE)
1417 * pkts should be refilled
1418 */
1419 assert_int_equal(efa_rdm_ep->efa_rx_pkts_to_post, 0);
1420 assert_int_equal(efa_rdm_ep->efa_rx_pkts_posted, rx_size + i);
1421
1422 /* recover the original value */
1423 efa_env.internal_rx_refill_threshold = threshold_orig;
1424 }
1425
1426 void test_efa_rdm_ep_rx_refill_threshold_smaller_than_rx_size(struct efa_resource **state)
1427 {
1428 test_efa_rdm_ep_rx_refill_impl(state, 8, 64);
1429 }
1430
1431 void test_efa_rdm_ep_rx_refill_threshold_larger_than_rx_size(struct efa_resource **state)
1432 {
1433 test_efa_rdm_ep_rx_refill_impl(state, 128, 64);
1434 }
1435
1436 /**
1437 * @brief when unsolicited write recv is supported (by device + env),
1438 * efa_rdm_ep_support_unsolicited_write_recv
1439 * should return true, otherwise it should return false
1440 *
1441 * @param[in] state struct efa_resource that is managed by the framework
1442 * @param[in] is_supported support status
1443 */
1444 void test_efa_rdm_ep_support_unsolicited_write_recv(struct efa_resource **state)
1445 {
1446 struct efa_rdm_ep *efa_rdm_ep;
1447 struct efa_resource *resource = *state;
1448
1449 efa_unit_test_resource_construct(resource, FI_EP_RDM, EFA_FABRIC_NAME);
1450
1451 efa_rdm_ep = container_of(resource->ep, struct efa_rdm_ep, base_ep.util_ep.ep_fid);
1452
1453 assert_int_equal(efa_use_unsolicited_write_recv(),
1454 efa_rdm_ep_support_unsolicited_write_recv(efa_rdm_ep));
1455 }
1456
1457 /**
1458 * @brief Test the default operational sizes for efa_rdm_ep
1459 *
1460 * @param state
1461 */
1462 void test_efa_rdm_ep_default_sizes(struct efa_resource **state)
1463 {
1464 struct efa_rdm_ep *efa_rdm_ep;
1465 struct efa_resource *resource = *state;
1466
1467 efa_unit_test_resource_construct(resource, FI_EP_RDM, EFA_FABRIC_NAME);
1468
1469 efa_rdm_ep = container_of(resource->ep, struct efa_rdm_ep, base_ep.util_ep.ep_fid);
1470
1471 /* sizes shared with base_ep */
1472 assert_int_equal(efa_rdm_ep->base_ep.max_msg_size, resource->info->ep_attr->max_msg_size);
1473 assert_int_equal(efa_rdm_ep->base_ep.max_rma_size, resource->info->ep_attr->max_msg_size);
1474 assert_int_equal(efa_rdm_ep->base_ep.inject_msg_size, resource->info->tx_attr->inject_size);
1475 assert_int_equal(efa_rdm_ep->base_ep.inject_rma_size, resource->info->tx_attr->inject_size);
1476 assert_int_equal(efa_rdm_ep->base_ep.rnr_retry, EFA_RDM_DEFAULT_RNR_RETRY);
1477
1478 /* efa_rdm_ep's own fields */
1479 assert_int_equal(efa_rdm_ep->max_tagged_size, resource->info->ep_attr->max_msg_size);
1480 assert_int_equal(efa_rdm_ep->max_atomic_size, resource->info->ep_attr->max_msg_size);
1481 assert_int_equal(efa_rdm_ep->inject_tagged_size, resource->info->tx_attr->inject_size);
1482 assert_int_equal(efa_rdm_ep->inject_atomic_size, resource->info->tx_attr->inject_size);
1483 }
1484
1485 /**
1486 * @brief Test the fi_endpoint API for efa_ep
1487 * for rdm ep type (because the dgram ep type should
1488 * have the same logic)
1489 * @param state
1490 */
1491 void test_efa_ep_open(struct efa_resource **state)
1492 {
1493 struct efa_resource *resource = *state;
1494 struct efa_base_ep *efa_ep;
1495 struct efa_domain *efa_domain;
1496
1497 efa_unit_test_resource_construct_ep_not_enabled(resource, FI_EP_RDM, EFA_DIRECT_FABRIC_NAME);
1498
1499 efa_ep = container_of(resource->ep, struct efa_base_ep, util_ep.ep_fid);
1500 efa_domain = container_of(resource->domain, struct efa_domain,
1501 util_domain.domain_fid);
1502
1503 /* Check various size limits defaults */
1504 assert_true(efa_ep->max_msg_size == efa_domain->device->ibv_port_attr.max_msg_sz);
1505 assert_true(efa_ep->max_rma_size == efa_domain->device->max_rdma_size);
1506 assert_true(efa_ep->inject_msg_size == efa_domain->device->efa_attr.inline_buf_size);
1507 /* TODO: update inject_rma_size to inline size after firmware
1508 * supports inline rdma write */
1509 assert_true(efa_ep->inject_rma_size == 0);
1510 assert_int_equal(efa_ep->rnr_retry, EFA_RNR_INFINITE_RETRY);
1511 }
1512
1513 /**
1514 * @brief Test the fi_cancel API for efa_ep
1515 * (for rdm ep type because dgram logic should be the same)
1516 * It should return -FI_ENOSYS as device doesn't support it;
1517 * @param state
1518 */
1519 void test_efa_ep_cancel(struct efa_resource **state)
1520 {
1521 struct efa_resource *resource = *state;
1522 int ret;
1523
1524 efa_unit_test_resource_construct_ep_not_enabled(resource, FI_EP_RDM, EFA_DIRECT_FABRIC_NAME);
1525
1526 ret = fi_cancel((struct fid *)resource->ep, NULL);
1527 assert_int_equal(ret, -FI_ENOSYS);
1528 }
1529
1530 /**
1531 * @brief Test the fi_getopt API fo efa_ep
1532 *
1533 * @param state
1534 */
1535 void test_efa_ep_getopt(struct efa_resource **state)
1536 {
1537 struct efa_resource *resource = *state;
1538 int optval_int;
1539 bool optval_bool;
1540 size_t optval_size_t;
1541 size_t optlen;
1542 struct efa_base_ep *efa_ep;
1543
1544 efa_unit_test_resource_construct_ep_not_enabled(resource, FI_EP_RDM, EFA_DIRECT_FABRIC_NAME);
1545
1546 efa_ep = container_of(resource->ep, struct efa_base_ep, util_ep.ep_fid);
1547
1548 optlen = sizeof(optval_int);
1549 assert_int_equal(fi_getopt(&resource->ep->fid, FI_OPT_ENDPOINT, FI_OPT_FI_HMEM_P2P, &optval_int, &optlen), 0);
1550 assert_int_equal(optval_int, FI_HMEM_P2P_REQUIRED);
1551
1552 optlen = sizeof(optval_bool);
1553
1554 assert_int_equal(fi_getopt(&resource->ep->fid, FI_OPT_ENDPOINT, FI_OPT_EFA_EMULATED_READ, &optval_bool, &optlen), 0);
1555 assert_false(optval_bool);
1556
1557 assert_int_equal(fi_getopt(&resource->ep->fid, FI_OPT_ENDPOINT, FI_OPT_EFA_EMULATED_WRITE, &optval_bool, &optlen), 0);
1558 assert_false(optval_bool);
1559
1560 optlen = sizeof(optval_size_t);
1561 assert_int_equal(fi_getopt(&resource->ep->fid, FI_OPT_ENDPOINT, FI_OPT_EFA_RNR_RETRY, &optval_size_t, &optlen), 0);
1562 assert_int_equal(optval_size_t, efa_ep->rnr_retry);
1563
1564 assert_int_equal(fi_getopt(&resource->ep->fid, FI_OPT_ENDPOINT, FI_OPT_MAX_MSG_SIZE, &optval_size_t, &optlen), 0);
1565 assert_int_equal(optval_size_t, efa_ep->max_msg_size);
1566
1567 assert_int_equal(fi_getopt(&resource->ep->fid, FI_OPT_ENDPOINT, FI_OPT_MAX_RMA_SIZE, &optval_size_t, &optlen), 0);
1568 assert_int_equal(optval_size_t, efa_ep->max_rma_size);
1569
1570 assert_int_equal(fi_getopt(&resource->ep->fid, FI_OPT_ENDPOINT, FI_OPT_INJECT_MSG_SIZE, &optval_size_t, &optlen), 0);
1571 assert_int_equal(optval_size_t, efa_ep->inject_msg_size);
1572
1573 assert_int_equal(fi_getopt(&resource->ep->fid, FI_OPT_ENDPOINT, FI_OPT_INJECT_RMA_SIZE, &optval_size_t, &optlen), 0);
1574 assert_int_equal(optval_size_t, efa_ep->inject_rma_size);
1575 }
1576
1577 /**
1578 * @brief Test the fi_setopt API for efa_ep
1579 * When RMA is requested, FI_OPT_EFA_USE_DEVICE_RDMA
1580 * cannot be set as false
1581 * @param state
1582 */
1583 void test_efa_ep_setopt_use_device_rdma(struct efa_resource **state)
1584 {
1585 struct efa_resource *resource = *state;
1586 bool optval;
1587 struct efa_base_ep *efa_ep;
1588
1589 efa_unit_test_resource_construct_ep_not_enabled(resource, FI_EP_RDM, EFA_DIRECT_FABRIC_NAME);
1590
1591 efa_ep = container_of(resource->ep, struct efa_base_ep, util_ep.ep_fid);
1592
1593 /* Hard code RMA caps in ep->info for local testing purpose */
1594 efa_ep->info->caps |= FI_RMA;
1595
1596 /* Disable rdma is not allowed when user requests FI_RMA */
1597 optval = false;
1598 assert_int_equal(fi_setopt(&resource->ep->fid, FI_OPT_ENDPOINT, FI_OPT_EFA_USE_DEVICE_RDMA, &optval, sizeof(optval)), -FI_EOPNOTSUPP);
1599 }
1600
1601 /**
1602 * @brief Test the fi_setopt API for efa_ep
1603 * FI_OPT_FI_HMEM_P2P cannot be set as FI_HMEM_P2P_DISABLED
1604 * @param state
1605 */
1606 void test_efa_ep_setopt_hmem_p2p(struct efa_resource **state)
1607 {
1608 struct efa_resource *resource = *state;
1609 int optval;
1610 int optvals[] = {
1611 FI_HMEM_P2P_DISABLED,
1612 FI_HMEM_P2P_ENABLED,
1613 FI_HMEM_P2P_PREFERRED,
1614 FI_HMEM_P2P_REQUIRED,
1615 };
1616 size_t num_optvals = sizeof(optvals) / sizeof(int);
1617 int i, expected_return;
1618
1619 efa_unit_test_resource_construct_ep_not_enabled(resource, FI_EP_RDM, EFA_DIRECT_FABRIC_NAME);
1620
1621 /* FI_HMEM_P2P_DISABLED is not allowed */
1622 for (i = 0; i < num_optvals; i++) {
1623 optval = optvals[i];
1624 expected_return = (optval == FI_HMEM_P2P_DISABLED) ? -FI_EOPNOTSUPP : FI_SUCCESS;
1625 assert_int_equal(fi_setopt(&resource->ep->fid, FI_OPT_ENDPOINT, FI_OPT_FI_HMEM_P2P, &optval, sizeof(optval)), expected_return);
1626 }
1627 }
1628
1629 /**
1630 * @brief Test the fi_setopt API for efa_ep with FI_OPT_EFA_RNR_RETRY
1631 * @param state
1632 */
1633 void test_efa_ep_setopt_rnr_retry(struct efa_resource **state)
1634 {
1635 struct efa_resource *resource = *state;
1636 size_t optval;
1637 struct efa_base_ep *efa_ep;
1638
1639 efa_unit_test_resource_construct_ep_not_enabled(resource, FI_EP_RDM, EFA_DIRECT_FABRIC_NAME);
1640
1641 efa_ep = container_of(resource->ep, struct efa_base_ep, util_ep.ep_fid);
1642 assert_false(efa_ep->efa_qp_enabled);
1643
1644 optval = 7;
1645 assert_int_equal(fi_setopt(&resource->ep->fid, FI_OPT_ENDPOINT, FI_OPT_EFA_RNR_RETRY, &optval, sizeof(optval)), FI_SUCCESS);
1646 assert_int_equal(efa_ep->rnr_retry, optval);
1647
1648 /* hack qp enabled status to allow local test */
1649 efa_ep->efa_qp_enabled = true;
1650 /* fi_setopt should fail when it's called after ep enable */
1651 assert_int_equal(fi_setopt(&resource->ep->fid, FI_OPT_ENDPOINT, FI_OPT_EFA_RNR_RETRY, &optval, sizeof(optval)), -FI_EINVAL);
1652 /* recover */
1653 efa_ep->efa_qp_enabled = false;
1654 }
1655
1656 /**
1657 * @brief Test the fi_setopt API for efa_ep with FI_OPT_*_SIZE
1658 * @param state
1659 */
1660 void test_efa_ep_setopt_sizes(struct efa_resource **state)
1661 {
1662 struct efa_resource *resource = *state;
1663 size_t optval;
1664 struct efa_base_ep *efa_ep;
1665
1666 efa_unit_test_resource_construct_ep_not_enabled(resource, FI_EP_RDM, EFA_DIRECT_FABRIC_NAME);
1667
1668 efa_ep = container_of(resource->ep, struct efa_base_ep, util_ep.ep_fid);
1669
1670 size_t size_thresholds[] = {
1671 [FI_OPT_MAX_MSG_SIZE] = (size_t) efa_ep->domain->device->ibv_port_attr.max_msg_sz,
1672 [FI_OPT_MAX_RMA_SIZE] = (size_t) efa_ep->domain->device->max_rdma_size,
1673 [FI_OPT_INJECT_MSG_SIZE] = (size_t) efa_ep->domain->device->efa_attr.inline_buf_size,
1674 [FI_OPT_INJECT_RMA_SIZE] = (size_t) 0,
1675 };
1676 int optnames[] = {
1677 FI_OPT_MAX_MSG_SIZE,
1678 FI_OPT_MAX_RMA_SIZE,
1679 FI_OPT_INJECT_MSG_SIZE,
1680 FI_OPT_INJECT_RMA_SIZE,
1681 };
1682 size_t num_optnames = sizeof(optnames) / sizeof(int);
1683 int i, optname;
1684
1685 for (i = 0; i < num_optnames; i++) {
1686 optname = optnames[i];
1687
1688 /* set optval <= threshold is allowed */
1689 optval = 0.5 * size_thresholds[optname];
1690 assert_int_equal(fi_setopt(&resource->ep->fid, FI_OPT_ENDPOINT, optname, &optval, sizeof(optval)), FI_SUCCESS);
1691
1692 /* set optval > threshold is NOT allowed */
1693 optval = size_thresholds[optname] + 10;
1694 assert_int_equal(fi_setopt(&resource->ep->fid, FI_OPT_ENDPOINT, optname, &optval, sizeof(optval)), -FI_EINVAL);
1695 }
1696 }
1697
1698 /**
1699 * @brief Test fi_ep_bind and fi_enable API for efa_ep
1700 *
1701 * @param state
1702 */
1703 void test_efa_ep_bind_and_enable(struct efa_resource **state)
1704 {
1705 struct efa_resource *resource = *state;
1706 struct efa_base_ep *efa_ep;
1707
1708 efa_unit_test_resource_construct(resource, FI_EP_RDM, EFA_DIRECT_FABRIC_NAME);
1709
1710 efa_ep = container_of(resource->ep, struct efa_base_ep, util_ep.ep_fid);
1711
1712 assert_true(efa_ep->efa_qp_enabled);
1713 /* we shouldn't have user recv qp for efa-direct */
1714 assert_true(efa_ep->user_recv_qp == NULL);
1715 }