Debian Codegraph @

prov/efa/test/efa_unit_test_cq.c

raw
1 /* SPDX-License-Identifier: BSD-2-Clause OR GPL-2.0-only */
2 /* SPDX-FileCopyrightText: Copyright Amazon.com, Inc. or its affiliates. All rights reserved. */
3
4 #include "efa_unit_tests.h"
5 #include "rdm/efa_rdm_cq.h"
6 #include "efa_av.h"
7
8 /**
9 * @brief implementation of test cases for fi_cq_read() works with empty device CQ for given endpoint type
10 *
11 * When CQ is empty, fi_cq_read() should return -FI_EAGAIN.
12 *
13 * @param[in] resource struct efa_resource that is managed by the framework
14 * @param[in] ep_type endpoint type, can be FI_EP_DGRAM or FI_EP_RDM
15 */
16 static
17 void test_impl_cq_read_empty_cq(struct efa_resource *resource, enum fi_ep_type ep_type)
18 {
19 struct ibv_cq_ex *ibv_cqx;
20 struct fi_cq_data_entry cq_entry;
21 int ret;
22 struct efa_base_ep *efa_base_ep;
23
24 efa_unit_test_resource_construct(resource, ep_type, EFA_FABRIC_NAME);
25
26 efa_base_ep = container_of(resource->ep, struct efa_base_ep, util_ep.ep_fid);
27 ibv_cqx = container_of(efa_base_ep->util_ep.rx_cq, struct efa_cq, util_cq)->ibv_cq.ibv_cq_ex;
28 ibv_cqx->start_poll = &efa_mock_ibv_start_poll_return_mock;
29
30 /* ibv_start_poll to return ENOENT means device CQ is empty */
31 will_return(efa_mock_ibv_start_poll_return_mock, ENOENT);
32
33 ret = fi_cq_read(resource->cq, &cq_entry, 1);
34
35 assert_int_equal(ret, -FI_EAGAIN);
36 }
37
38 /**
39 * @brief verify DGRAM CQ's fi_cq_read() works with empty CQ
40 *
41 * When CQ is empty, fi_cq_read() should return -FI_EAGAIN.
42 *
43 * @param[in] state struct efa_resource that is managed by the framework
44 */
45 void test_dgram_cq_read_empty_cq(struct efa_resource **state)
46 {
47 struct efa_resource *resource = *state;
48 test_impl_cq_read_empty_cq(resource, FI_EP_DGRAM);
49 }
50
51 /**
52 * @brief verify RDM CQ's fi_cq_read() works with empty CQ
53 *
54 * When CQ is empty, fi_cq_read() should return -FI_EAGAIN.
55 *
56 * @param[in] state struct efa_resource that is managed by the framework
57 */
58 void test_ibv_cq_ex_read_empty_cq(struct efa_resource **state)
59 {
60 struct efa_resource *resource = *state;
61 test_impl_cq_read_empty_cq(resource, FI_EP_RDM);
62 }
63
64 /**
65 * @brief test RDM CQ's fi_cq_read()/fi_cq_readerr() works properly when rdma-core return bad status for send.
66 *
67 * When the send operation failed, fi_cq_read() should return -FI_EAVAIL, which means error available.
68 * then user should call fi_cq_readerr() to get an error CQ entry that contain error code.
69 *
70 * @param[in] state struct efa_resource that is managed by the framework
71 * @param[in] local_host_id Local(sender) host id
72 * @param[in] peer_host_id Peer(receiver) host id
73 * @param[in] vendor_error Vendor error returned by ibv_read_vendor_err
74 */
75 static void test_rdm_cq_read_bad_send_status(struct efa_resource *resource,
76 uint64_t local_host_id, uint64_t peer_host_id,
77 int vendor_error)
78 {
79 const char *strerror;
80 fi_addr_t addr;
81 int ret, err;
82 char host_id_str[] = "xxxxx host id: i-01234567812345678";
83 size_t raw_addr_len = sizeof(struct efa_ep_addr);
84 struct efa_ep_addr raw_addr;
85 struct efa_unit_test_buff send_buff;
86 struct fi_cq_data_entry cq_entry;
87 struct fi_cq_err_entry cq_err_entry = {0};
88 struct ibv_cq_ex *ibv_cqx;
89 struct ibv_qp_ex *ibv_qpx;
90 struct efa_rdm_ep *efa_rdm_ep;
91 struct efa_rdm_peer *peer;
92 struct efa_rdm_cq *efa_rdm_cq;
93
94 /* disable shm to force using efa device to send */
95 efa_unit_test_resource_construct_rdm_shm_disabled(resource);
96 efa_unit_test_buff_construct(&send_buff, resource, 4096 /* buff_size */);
97
98 efa_rdm_ep = container_of(resource->ep, struct efa_rdm_ep, base_ep.util_ep.ep_fid);
99 efa_rdm_ep->host_id = local_host_id;
100 ibv_qpx = efa_rdm_ep->base_ep.qp->ibv_qp_ex;
101
102 efa_rdm_cq = container_of(resource->cq, struct efa_rdm_cq, efa_cq.util_cq.cq_fid.fid);
103 ibv_cqx = efa_rdm_cq->efa_cq.ibv_cq.ibv_cq_ex;
104
105 ret = fi_getname(&resource->ep->fid, &raw_addr, &raw_addr_len);
106 assert_int_equal(ret, 0);
107 raw_addr.qpn = 1;
108 raw_addr.qkey = 0x1234;
109 ret = fi_av_insert(resource->av, &raw_addr, 1, &addr, 0 /* flags */, NULL /* context */);
110 assert_int_equal(ret, 1);
111
112 peer = efa_rdm_ep_get_peer(efa_rdm_ep, addr);
113 assert_non_null(peer);
114 peer->host_id = peer_host_id;
115
116 ibv_qpx->wr_start = &efa_mock_ibv_wr_start_no_op;
117 /* this mock will save the send work request (wr) in a global list */
118 ibv_qpx->wr_send = &efa_mock_ibv_wr_send_save_wr;
119 ibv_qpx->wr_set_sge_list = &efa_mock_ibv_wr_set_sge_list_no_op;
120 ibv_qpx->wr_set_ud_addr = &efa_mock_ibv_wr_set_ud_addr_no_op;
121 ibv_qpx->wr_complete = &efa_mock_ibv_wr_complete_no_op;
122 assert_int_equal(g_ibv_submitted_wr_id_cnt, 0);
123
124 err = fi_send(resource->ep, send_buff.buff, send_buff.size, fi_mr_desc(send_buff.mr), addr, NULL /* context */);
125 assert_int_equal(err, 0);
126 /* fi_send() called efa_mock_ibv_wr_send_save_wr(), which saved one send_wr in g_ibv_submitted_wr_id_vec */
127 assert_int_equal(g_ibv_submitted_wr_id_cnt, 1);
128
129 /* this mock will set ibv_cq_ex->wr_id to the wr_id f the head of global send_wr,
130 * and set ibv_cq_ex->status to mock value */
131 ibv_cqx->start_poll = &efa_mock_ibv_start_poll_use_saved_send_wr_with_mock_status;
132 ibv_cqx->end_poll = &efa_mock_ibv_end_poll_check_mock;
133 ibv_cqx->read_opcode = &efa_mock_ibv_read_opcode_return_mock;
134 ibv_cqx->read_vendor_err = &efa_mock_ibv_read_vendor_err_return_mock;
135 ibv_cqx->read_qp_num = &efa_mock_ibv_read_qp_num_return_mock;
136 will_return(efa_mock_ibv_start_poll_use_saved_send_wr_with_mock_status, IBV_WC_GENERAL_ERR);
137 will_return(efa_mock_ibv_end_poll_check_mock, NULL);
138 will_return(efa_mock_ibv_read_opcode_return_mock, IBV_WC_SEND);
139 will_return(efa_mock_ibv_read_vendor_err_return_mock, vendor_error);
140 will_return(efa_mock_ibv_read_qp_num_return_mock, efa_rdm_ep->base_ep.qp->qp_num);
141 ret = fi_cq_read(resource->cq, &cq_entry, 1);
142 /* fi_cq_read() called efa_mock_ibv_start_poll_use_saved_send_wr(), which pulled one send_wr from g_ibv_submitted_wr_idv=_vec */
143 assert_int_equal(g_ibv_submitted_wr_id_cnt, 0);
144 assert_int_equal(ret, -FI_EAVAIL);
145
146 /* Allocate memory to read CQ error */
147 cq_err_entry.err_data_size = EFA_ERROR_MSG_BUFFER_LENGTH;
148 cq_err_entry.err_data = malloc(cq_err_entry.err_data_size);
149 assert_non_null(cq_err_entry.err_data);
150
151 ret = fi_cq_readerr(resource->cq, &cq_err_entry, 0);
152 assert_true(cq_err_entry.err_data_size > 0);
153 strerror = fi_cq_strerror(resource->cq, cq_err_entry.prov_errno, cq_err_entry.err_data, NULL, 0);
154
155 assert_int_equal(ret, 1);
156 assert_int_not_equal(cq_err_entry.err, FI_SUCCESS);
157 assert_int_equal(cq_err_entry.prov_errno, vendor_error);
158
159 /* Reset value */
160 memset(host_id_str, 0, sizeof(host_id_str));
161
162 /* Set expected host id */
163 if (local_host_id) {
164 snprintf(host_id_str, sizeof(host_id_str), "My host id: i-%017lx", local_host_id);
165 } else {
166 strcpy(host_id_str, "My host id: N/A");
167 }
168 /* Look for My host id */
169 assert_non_null(strstr(strerror, host_id_str));
170
171 /* Reset value */
172 memset(host_id_str, 0, sizeof(host_id_str));
173
174 /* Set expected host id */
175 if (peer_host_id) {
176 snprintf(host_id_str, sizeof(host_id_str), "Peer host id: i-%017lx", peer_host_id);
177 } else {
178 strcpy(host_id_str, "Peer host id: N/A");
179 }
180 /* Look for peer host id */
181 assert_non_null(strstr(strerror, host_id_str));
182 efa_unit_test_buff_destruct(&send_buff);
183 }
184
185 /**
186 * @brief test that RDM CQ's fi_cq_read()/fi_cq_readerr() works properly when rdma-core returns
187 * unresponsive receiver error for send.
188 *
189 * When send operation failed, fi_cq_read() should return -FI_EAVAIL, which means error available.
190 * then user should call fi_cq_readerr() to get an error CQ entry that contain error code.
191 *
192 * @param[in] state struct efa_resource that is managed by the framework
193 */
194 void test_rdm_cq_read_bad_send_status_unresponsive_receiver(struct efa_resource **state)
195 {
196 struct efa_resource *resource = *state;
197 test_rdm_cq_read_bad_send_status(resource,
198 0x1234567812345678, 0x8765432187654321,
199 EFA_IO_COMP_STATUS_LOCAL_ERROR_UNRESP_REMOTE);
200 }
201
202 /**
203 * @brief test that RDM CQ's fi_cq_read()/fi_cq_readerr() works properly when rdma-core returns
204 * unresponsive receiver error for send. This test verifies peer host id is printed correctly if it is unknown.
205 *
206 * When send operation failed, fi_cq_read() should return -FI_EAVAIL, which means error available.
207 * then user should call fi_cq_readerr() to get an error CQ entry that contain error code.
208 *
209 * @param[in] state struct efa_resource that is managed by the framework
210 */
211 void test_rdm_cq_read_bad_send_status_unresponsive_receiver_missing_peer_host_id(struct efa_resource **state)
212 {
213 struct efa_resource *resource = *state;
214 test_rdm_cq_read_bad_send_status(resource,
215 0x1234567812345678, 0,
216 EFA_IO_COMP_STATUS_LOCAL_ERROR_UNRESP_REMOTE);
217 }
218
219 /**
220 * @brief test that RDM CQ's fi_cq_read()/fi_cq_readerr() works properly when rdma-core returns
221 * unreachable remote error for send.
222 *
223 * When send operation failed, fi_cq_read() should return -FI_EAVAIL, which means error available.
224 * then user should call fi_cq_readerr() to get an error CQ entry that contain error code.
225 *
226 * @param[in] state struct efa_resource that is managed by the framework
227 */
228 void test_rdm_cq_read_bad_send_status_unreachable_receiver(struct efa_resource **state)
229 {
230 struct efa_resource *resource = *state;
231 test_rdm_cq_read_bad_send_status(resource,
232 0x1234567812345678, 0x8765432187654321,
233 EFA_IO_COMP_STATUS_LOCAL_ERROR_UNREACH_REMOTE);
234 }
235
236 /**
237 * @brief test that RDM CQ's fi_cq_read()/fi_cq_readerr() works properly when rdma-core returns
238 * invalid qpn error for send.
239 *
240 * When send operation failed, fi_cq_read() should return -FI_EAVAIL, which means error available.
241 * then user should call fi_cq_readerr() to get an error CQ entry that contain error code.
242 *
243 * @param[in] state struct efa_resource that is managed by the framework
244 */
245 void test_rdm_cq_read_bad_send_status_invalid_qpn(struct efa_resource **state)
246 {
247 struct efa_resource *resource = *state;
248
249 test_rdm_cq_read_bad_send_status(resource,
250 0x1234567812345678, 0x8765432187654321,
251 EFA_IO_COMP_STATUS_REMOTE_ERROR_BAD_DEST_QPN);
252 }
253
254 /**
255 * @brief test that RDM CQ's fi_cq_read()/fi_cq_readerr() works properly when rdma-core returns
256 * message too long error for send.
257 *
258 * When send operation failed, fi_cq_read() should return -FI_EAVAIL, which means error available.
259 * then user should call fi_cq_readerr() to get an error CQ entry that contain error code.
260 *
261 * @param[in] state struct efa_resource that is managed by the framework
262 */
263 void test_rdm_cq_read_bad_send_status_message_too_long(struct efa_resource **state)
264 {
265 struct efa_resource *resource = *state;
266 test_rdm_cq_read_bad_send_status(resource,
267 0x1234567812345678, 0x8765432187654321,
268 EFA_IO_COMP_STATUS_LOCAL_ERROR_BAD_LENGTH);
269 }
270
271 /**
272 * @brief verify that fi_cq_read/fi_cq_readerr works properly when rdma-core return bad status for recv.
273 *
274 * When an ibv_post_recv() operation failed, no data was received. Therefore libfabric cannot
275 * find the corresponding RX operation to write a CQ error. It will write an EQ error instead.
276 *
277 * @param[in] state struct efa_resource that is managed by the framework
278 */
279 void test_ibv_cq_ex_read_bad_recv_status(struct efa_resource **state)
280 {
281 struct efa_rdm_ep *efa_rdm_ep;
282 struct efa_resource *resource = *state;
283 struct efa_rdm_pke *pkt_entry;
284 struct fi_cq_data_entry cq_entry;
285 struct fi_eq_err_entry eq_err_entry;
286 int ret;
287 struct efa_rdm_cq *efa_rdm_cq;
288 struct ibv_cq_ex *ibv_cqx;
289
290
291 efa_unit_test_resource_construct(resource, FI_EP_RDM, EFA_FABRIC_NAME);
292 efa_rdm_ep = container_of(resource->ep, struct efa_rdm_ep, base_ep.util_ep.ep_fid);
293
294 /*
295 * The rx pkt entry should only be allocated and posted by the progress engine.
296 * However, to mock a receive completion, we have to allocate an rx entry
297 * and modify it out of band. The proess engine grow the rx pool in the first
298 * call and set efa_rdm_ep->efa_rx_pkts_posted as the rx pool size. Here we
299 * follow the progress engine to set the efa_rx_pkts_posted counter manually
300 * TODO: modify the rx pkt as part of the ibv cq poll mock so we don't have to
301 * allocate pkt entry and hack the pkt counters.
302 */
303 pkt_entry = efa_rdm_pke_alloc(efa_rdm_ep, efa_rdm_ep->efa_rx_pkt_pool, EFA_RDM_PKE_FROM_EFA_RX_POOL);
304 assert_non_null(pkt_entry);
305 efa_rdm_ep->efa_rx_pkts_posted = efa_rdm_ep_get_rx_pool_size(efa_rdm_ep);
306
307 efa_rdm_cq = container_of(resource->cq, struct efa_rdm_cq, efa_cq.util_cq.cq_fid.fid);
308 ibv_cqx = efa_rdm_cq->efa_cq.ibv_cq.ibv_cq_ex;
309
310 ibv_cqx->start_poll = &efa_mock_ibv_start_poll_return_mock;
311 ibv_cqx->end_poll = &efa_mock_ibv_end_poll_check_mock;
312 ibv_cqx->read_opcode = &efa_mock_ibv_read_opcode_return_mock;
313 ibv_cqx->read_vendor_err = &efa_mock_ibv_read_vendor_err_return_mock;
314 ibv_cqx->read_qp_num = &efa_mock_ibv_read_qp_num_return_mock;
315
316 will_return(efa_mock_ibv_start_poll_return_mock, 0);
317 will_return(efa_mock_ibv_end_poll_check_mock, NULL);
318 /* efa_mock_ibv_read_opcode_return_mock() will be called once in release mode,
319 * but will be called twice in debug mode. because there is an assertion that called ibv_read_opcode(),
320 * therefore use will_return_always()
321 */
322 will_return_always(efa_mock_ibv_read_opcode_return_mock, IBV_WC_RECV);
323 will_return_always(efa_mock_ibv_read_qp_num_return_mock, efa_rdm_ep->base_ep.qp->qp_num);
324 will_return(efa_mock_ibv_read_vendor_err_return_mock, EFA_IO_COMP_STATUS_LOCAL_ERROR_UNRESP_REMOTE);
325 /* the recv error will not populate to application cq because it's an EFA internal error and
326 * and not related to any application recv. Currently we can only read the error from eq.
327 */
328 ibv_cqx->wr_id = (uintptr_t)pkt_entry;
329 ibv_cqx->status = IBV_WC_GENERAL_ERR;
330
331 #if HAVE_CAPS_UNSOLICITED_WRITE_RECV
332 if (efa_use_unsolicited_write_recv()) {
333 efadv_cq_from_ibv_cq_ex(ibv_cqx)->wc_is_unsolicited = &efa_mock_efadv_wc_is_unsolicited;
334 will_return(efa_mock_efadv_wc_is_unsolicited, false);
335 }
336 #endif
337
338 ret = fi_cq_read(resource->cq, &cq_entry, 1);
339 assert_int_equal(ret, -FI_EAGAIN);
340
341 ret = fi_eq_readerr(resource->eq, &eq_err_entry, 0);
342 assert_int_equal(ret, sizeof(eq_err_entry));
343 assert_int_not_equal(eq_err_entry.err, FI_SUCCESS);
344 assert_int_equal(eq_err_entry.prov_errno, EFA_IO_COMP_STATUS_LOCAL_ERROR_UNRESP_REMOTE);
345 }
346
347 /**
348 * @brief verify that fi_cq_read/fi_eq_read works properly when rdma-core return bad status for
349 * recv rdma with imm.
350 *
351 * When getting a wc error of op code IBV_WC_RECV_RDMA_WITH_IMM, libfabric cannot find the
352 * corresponding application operation to write a cq error.
353 * It will write an EQ error instead.
354 *
355 * @param[in] state struct efa_resource that is managed by the framework
356 * @param[in] use_unsolicited_recv whether to use unsolicited write recv
357 */
358 void test_ibv_cq_ex_read_bad_recv_rdma_with_imm_status_impl(struct efa_resource **state, bool use_unsolicited_recv)
359 {
360 struct efa_rdm_ep *efa_rdm_ep;
361 struct efa_resource *resource = *state;
362 struct fi_cq_data_entry cq_entry;
363 struct fi_eq_err_entry eq_err_entry;
364 int ret;
365 struct efa_rdm_cq *efa_rdm_cq;
366 struct ibv_cq_ex *ibv_cqx;
367
368
369 efa_unit_test_resource_construct(resource, FI_EP_RDM, EFA_FABRIC_NAME);
370 efa_rdm_ep = container_of(resource->ep, struct efa_rdm_ep, base_ep.util_ep.ep_fid);
371
372 efa_rdm_cq = container_of(resource->cq, struct efa_rdm_cq, efa_cq.util_cq.cq_fid.fid);
373 ibv_cqx = efa_rdm_cq->efa_cq.ibv_cq.ibv_cq_ex;
374
375 ibv_cqx->start_poll = &efa_mock_ibv_start_poll_return_mock;
376 ibv_cqx->end_poll = &efa_mock_ibv_end_poll_check_mock;
377 ibv_cqx->read_opcode = &efa_mock_ibv_read_opcode_return_mock;
378 ibv_cqx->read_vendor_err = &efa_mock_ibv_read_vendor_err_return_mock;
379 ibv_cqx->read_qp_num = &efa_mock_ibv_read_qp_num_return_mock;
380
381 will_return(efa_mock_ibv_start_poll_return_mock, 0);
382 will_return(efa_mock_ibv_end_poll_check_mock, NULL);
383 /* efa_mock_ibv_read_opcode_return_mock() will be called once in release mode,
384 * but will be called twice in debug mode. because there is an assertion that called ibv_read_opcode(),
385 * therefore use will_return_always()
386 */
387 will_return_always(efa_mock_ibv_read_opcode_return_mock, IBV_WC_RECV_RDMA_WITH_IMM);
388 will_return_always(efa_mock_ibv_read_qp_num_return_mock, efa_rdm_ep->base_ep.qp->qp_num);
389 will_return(efa_mock_ibv_read_vendor_err_return_mock, EFA_IO_COMP_STATUS_FLUSHED);
390
391 g_efa_unit_test_mocks.efa_device_support_unsolicited_write_recv = &efa_mock_efa_device_support_unsolicited_write_recv;
392
393 #if HAVE_CAPS_UNSOLICITED_WRITE_RECV
394 if (use_unsolicited_recv) {
395 efadv_cq_from_ibv_cq_ex(ibv_cqx)->wc_is_unsolicited = &efa_mock_efadv_wc_is_unsolicited;
396 will_return(efa_mock_efa_device_support_unsolicited_write_recv, true);
397 will_return(efa_mock_efadv_wc_is_unsolicited, true);
398 ibv_cqx->wr_id = 0;
399 } else {
400 /*
401 * For solicited write recv, it will consume an internal rx pkt
402 */
403 will_return(efa_mock_efa_device_support_unsolicited_write_recv, false);
404 struct efa_rdm_pke *pkt_entry = efa_rdm_pke_alloc(efa_rdm_ep, efa_rdm_ep->efa_rx_pkt_pool, EFA_RDM_PKE_FROM_EFA_RX_POOL);
405 assert_non_null(pkt_entry);
406 efa_rdm_ep->efa_rx_pkts_posted = efa_rdm_ep_get_rx_pool_size(efa_rdm_ep);
407 ibv_cqx->wr_id = (uintptr_t)pkt_entry;
408 }
409 #else
410 /*
411 * Always test with solicited recv
412 */
413 will_return(efa_mock_efa_device_support_unsolicited_write_recv, false);
414 struct efa_rdm_pke *pkt_entry = efa_rdm_pke_alloc(efa_rdm_ep, efa_rdm_ep->efa_rx_pkt_pool, EFA_RDM_PKE_FROM_EFA_RX_POOL);
415 assert_non_null(pkt_entry);
416 efa_rdm_ep->efa_rx_pkts_posted = efa_rdm_ep_get_rx_pool_size(efa_rdm_ep);
417 ibv_cqx->wr_id = (uintptr_t)pkt_entry;
418 #endif
419 /* the recv rdma with imm will not populate to application cq because it's an EFA internal error and
420 * and not related to any application operations. Currently we can only read the error from eq.
421 */
422 ibv_cqx->status = IBV_WC_GENERAL_ERR;
423 ret = fi_cq_read(resource->cq, &cq_entry, 1);
424 assert_int_equal(ret, -FI_EAGAIN);
425
426 ret = fi_eq_readerr(resource->eq, &eq_err_entry, 0);
427 assert_int_equal(ret, sizeof(eq_err_entry));
428 assert_int_not_equal(eq_err_entry.err, FI_SUCCESS);
429 assert_int_equal(eq_err_entry.prov_errno, EFA_IO_COMP_STATUS_FLUSHED);
430 }
431
432 void test_ibv_cq_ex_read_bad_recv_rdma_with_imm_status_use_unsolicited_recv(struct efa_resource **state)
433 {
434 test_ibv_cq_ex_read_bad_recv_rdma_with_imm_status_impl(state, true);
435 }
436
437 void test_ibv_cq_ex_read_bad_recv_rdma_with_imm_status_use_solicited_recv(struct efa_resource **state)
438 {
439 test_ibv_cq_ex_read_bad_recv_rdma_with_imm_status_impl(state, false);
440 }
441
442 /**
443 * @brief verify that fi_cq_read/fi_cq_readerr works properly when ibv_start_poll failed.
444 *
445 * When an ibv_start_poll() failed. Libfabric should write an EQ error.
446 *
447 * @param[in] state struct efa_resource that is managed by the framework
448 */
449 void test_ibv_cq_ex_read_failed_poll(struct efa_resource **state)
450 {
451 struct efa_resource *resource = *state;
452 struct fi_cq_data_entry cq_entry;
453 struct fi_cq_err_entry cq_err_entry;
454 int ret;
455 struct efa_rdm_cq *efa_rdm_cq;
456 struct ibv_cq_ex *ibv_cqx;
457
458 efa_unit_test_resource_construct(resource, FI_EP_RDM, EFA_FABRIC_NAME);
459
460 efa_rdm_cq = container_of(resource->cq, struct efa_rdm_cq, efa_cq.util_cq.cq_fid.fid);
461 ibv_cqx = efa_rdm_cq->efa_cq.ibv_cq.ibv_cq_ex;
462
463 ibv_cqx->start_poll = &efa_mock_ibv_start_poll_return_mock;
464 ibv_cqx->end_poll = &efa_mock_ibv_end_poll_check_mock;
465 ibv_cqx->read_vendor_err = &efa_mock_ibv_read_vendor_err_return_mock;
466
467 will_return(efa_mock_ibv_start_poll_return_mock, EFAULT);
468 will_return(efa_mock_ibv_read_vendor_err_return_mock, EFA_IO_COMP_STATUS_LOCAL_ERROR_UNRESP_REMOTE);
469
470 ret = fi_cq_read(resource->cq, &cq_entry, 1);
471 assert_int_equal(ret, -FI_EAVAIL);
472
473 ret = fi_cq_readerr(resource->cq, &cq_err_entry, 0);
474 assert_int_equal(ret, 1);
475 assert_int_not_equal(cq_err_entry.err, FI_ENOENT);
476 assert_int_equal(cq_err_entry.prov_errno, EFA_IO_COMP_STATUS_LOCAL_ERROR_UNRESP_REMOTE);
477 }
478
479 /**
480 * @brief Test efa_rdm_cq_open() handles rdma-core CQ creation failure gracefully
481 *
482 * @param[in] state struct efa_resource that is managed by the framework
483 */
484 void test_rdm_cq_create_error_handling(struct efa_resource **state)
485 {
486
487 struct efa_resource *resource = *state;
488 struct ibv_device **ibv_device_list;
489 struct efa_device efa_device = {0};
490 struct efa_domain *efa_domain = NULL;
491 struct verbs_context *vctx = NULL;
492 struct fi_cq_attr cq_attr = {0};
493
494 ibv_device_list = ibv_get_device_list(&g_device_cnt);
495 if (ibv_device_list == NULL) {
496 skip();
497 return;
498 }
499 efa_device_construct(&efa_device, 0, ibv_device_list[0]);
500
501 resource->hints = efa_unit_test_alloc_hints(FI_EP_RDM, EFA_FABRIC_NAME);
502 assert_non_null(resource->hints);
503 assert_int_equal(fi_getinfo(FI_VERSION(1, 14), NULL, NULL, 0ULL, resource->hints, &resource->info), 0);
504 assert_int_equal(fi_fabric(resource->info->fabric_attr, &resource->fabric, NULL), 0);
505 assert_int_equal(fi_domain(resource->fabric, resource->info, &resource->domain, NULL), 0);
506
507 vctx = verbs_get_ctx_op(efa_device.ibv_ctx, create_cq_ex);
508 #if HAVE_EFADV_CQ_EX
509 g_efa_unit_test_mocks.efadv_create_cq = &efa_mock_efadv_create_cq_set_eopnotsupp_and_return_null;
510 expect_function_call(efa_mock_efadv_create_cq_set_eopnotsupp_and_return_null);
511 #endif
512 /* Mock out the create_cq_ex function pointer which is called by ibv_create_cq_ex */
513 vctx->create_cq_ex = &efa_mock_create_cq_ex_return_null;
514 expect_function_call(efa_mock_create_cq_ex_return_null);
515
516 efa_domain = container_of(resource->domain, struct efa_domain, util_domain.domain_fid);
517 efa_domain->device = &efa_device;
518
519 assert_int_not_equal(fi_cq_open(resource->domain, &cq_attr, &resource->cq, NULL), 0);
520 /* set cq as NULL to avoid double free by fi_close in cleanup stage */
521 resource->cq = NULL;
522 }
523
524 /**
525 * @brief get the length of the ibv_cq_poll_list for a given efa_rdm_cq
526 *
527 * @param cq_fid cq fid
528 * @return int the length of the ibv_cq_poll_list
529 */
530 static
531 int test_efa_rdm_cq_get_ibv_cq_poll_list_length(struct fid_cq *cq_fid)
532 {
533 struct efa_rdm_cq *cq;
534
535 cq = container_of(cq_fid, struct efa_rdm_cq, efa_cq.util_cq.cq_fid.fid);
536 return efa_unit_test_get_dlist_length(&cq->ibv_cq_poll_list);
537 }
538
539 /**
540 * @brief Check the length of ibv_cq_poll_list when 1 cq is bind to 1 ep
541 * as both tx/rx cq.
542 *
543 * @param state struct efa_resource that is managed by the framework
544 */
545 void test_efa_rdm_cq_ibv_cq_poll_list_same_tx_rx_cq_single_ep(struct efa_resource **state)
546 {
547 struct efa_resource *resource = *state;
548
549 efa_unit_test_resource_construct(resource, FI_EP_RDM, EFA_FABRIC_NAME);
550
551 /* efa_unit_test_resource_construct binds single OFI CQ as both tx/rx cq of ep */
552 assert_int_equal(test_efa_rdm_cq_get_ibv_cq_poll_list_length(resource->cq), 1);
553 }
554
555 /**
556 * @brief Check the length of ibv_cq_poll_list when separate tx/rx cq is bind to 1 ep.
557 *
558 * @param state struct efa_resource that is managed by the framework
559 */
560 void test_efa_rdm_cq_ibv_cq_poll_list_separate_tx_rx_cq_single_ep(struct efa_resource **state)
561 {
562 struct efa_resource *resource = *state;
563 struct fid_cq *txcq, *rxcq;
564 struct fi_cq_attr cq_attr = {0};
565
566 efa_unit_test_resource_construct_no_cq_and_ep_not_enabled(resource, FI_EP_RDM, EFA_FABRIC_NAME);
567
568 assert_int_equal(fi_cq_open(resource->domain, &cq_attr, &txcq, NULL), 0);
569
570 assert_int_equal(fi_ep_bind(resource->ep, &txcq->fid, FI_SEND), 0);
571
572 assert_int_equal(fi_cq_open(resource->domain, &cq_attr, &rxcq, NULL), 0);
573
574 assert_int_equal(fi_ep_bind(resource->ep, &rxcq->fid, FI_RECV), 0);
575
576 assert_int_equal(fi_enable(resource->ep), 0);
577
578 assert_int_equal(test_efa_rdm_cq_get_ibv_cq_poll_list_length(txcq), 2);
579
580 assert_int_equal(test_efa_rdm_cq_get_ibv_cq_poll_list_length(rxcq), 2);
581
582 /* ep must be closed before cq/av/eq... */
583 fi_close(&resource->ep->fid);
584 resource->ep = NULL;
585 fi_close(&txcq->fid);
586 fi_close(&rxcq->fid);
587 }
588
589 void test_efa_rdm_cq_post_initial_rx_pkts(struct efa_resource **state)
590 {
591 struct efa_resource *resource = *state;
592 struct efa_rdm_ep *efa_rdm_ep;
593 struct efa_rdm_cq *efa_rdm_cq;
594
595 efa_unit_test_resource_construct(resource, FI_EP_RDM, EFA_FABRIC_NAME);
596 efa_rdm_ep = container_of(resource->ep, struct efa_rdm_ep, base_ep.util_ep.ep_fid);
597 efa_rdm_cq = container_of(resource->cq, struct efa_rdm_cq, efa_cq.util_cq.cq_fid.fid);
598
599 /* At this time, rx pkts are not growed and posted */
600 assert_int_equal(efa_rdm_ep->efa_rx_pkts_to_post, 0);
601 assert_int_equal(efa_rdm_ep->efa_rx_pkts_posted, 0);
602 assert_int_equal(efa_rdm_ep->efa_rx_pkts_held, 0);
603
604 /* cq read need to scan the ep list since a ep is bind */
605 assert_true(efa_rdm_cq->need_to_scan_ep_list);
606 fi_cq_read(resource->cq, NULL, 0);
607
608 /* At this time, rx pool size number of rx pkts are posted */
609 assert_int_equal(efa_rdm_ep->efa_rx_pkts_posted, efa_rdm_ep_get_rx_pool_size(efa_rdm_ep));
610 assert_int_equal(efa_rdm_ep->efa_rx_pkts_to_post, 0);
611 assert_int_equal(efa_rdm_ep->efa_rx_pkts_held, 0);
612
613 /* scan is done */
614 assert_false(efa_rdm_cq->need_to_scan_ep_list);
615 }
616 #if HAVE_EFADV_CQ_EX
617 /**
618 * @brief Construct an RDM endpoint and receive an eager MSG RTM packet.
619 * Simulate EFA device by setting peer AH to unknown and make sure the
620 * endpoint recovers the peer address iff(if and only if) the peer is
621 * inserted to AV.
622 *
623 * @param resource struct efa_resource that is managed by the framework
624 * @param remove_peer Boolean value that indicates if the peer was removed explicitly
625 * @param support_efadv_cq Boolean value that indicates if EFA device supports EFA DV CQ
626 */
627 static void test_impl_ibv_cq_ex_read_unknow_peer_ah(struct efa_resource *resource, bool remove_peer, bool support_efadv_cq)
628 {
629 struct efa_rdm_ep *efa_rdm_ep;
630 struct efa_rdm_pke *pkt_entry;
631 struct efa_ep_addr raw_addr = {0};
632 size_t raw_addr_len = sizeof(raw_addr);
633 fi_addr_t peer_addr = 0;
634 struct fi_cq_data_entry cq_entry;
635 struct efa_unit_test_eager_rtm_pkt_attr pkt_attr = {0};
636 struct efadv_cq *efadv_cq;
637 struct efa_unit_test_buff recv_buff;
638 int ret;
639 struct efa_rdm_cq *efa_rdm_cq;
640 struct ibv_cq_ex *ibv_cqx;
641
642 /*
643 * Always use mocked efadv_create_cq instead of the real one.
644 * Otherwise the test is undeterministic depending on the host kernel:
645 * - If the kernel supports EFA DV CQ and we set support_efadv_cq = true, then the test will pass
646 * - If the kernel does NOT support EFA DV CQ and we set support_efadv_cq = true, then the test will fail
647 */
648 if (support_efadv_cq) {
649 g_efa_unit_test_mocks.efadv_create_cq = &efa_mock_efadv_create_cq_with_ibv_create_cq_ex;
650 expect_function_call(efa_mock_efadv_create_cq_with_ibv_create_cq_ex);
651 } else {
652 g_efa_unit_test_mocks.efadv_create_cq = &efa_mock_efadv_create_cq_set_eopnotsupp_and_return_null;
653 expect_function_call(efa_mock_efadv_create_cq_set_eopnotsupp_and_return_null);
654 }
655
656 efa_unit_test_resource_construct(resource, FI_EP_RDM, EFA_FABRIC_NAME);
657
658 efa_rdm_ep = container_of(resource->ep, struct efa_rdm_ep, base_ep.util_ep.ep_fid);
659 efa_rdm_cq = container_of(resource->cq, struct efa_rdm_cq, efa_cq.util_cq.cq_fid.fid);
660 ibv_cqx = efa_rdm_cq->efa_cq.ibv_cq.ibv_cq_ex;
661
662 /* Construct a minimal recv buffer */
663 efa_unit_test_buff_construct(&recv_buff, resource, efa_rdm_ep->min_multi_recv_size);
664
665 /* Create and register a fake peer */
666 ret = fi_getname(&resource->ep->fid, &raw_addr, &raw_addr_len);
667 assert_int_equal(ret, 0);
668 raw_addr.qpn = 0;
669 raw_addr.qkey = 0x1234;
670
671 struct efa_rdm_peer *peer;
672 ret = fi_av_insert(resource->av, &raw_addr, 1, &peer_addr, 0, NULL);
673 assert_int_equal(ret, 1);
674
675 /* Skip handshake */
676 peer = efa_rdm_ep_get_peer(efa_rdm_ep, peer_addr);
677 assert_non_null(peer);
678 peer->flags |= EFA_RDM_PEER_HANDSHAKE_SENT;
679
680 /*
681 * The rx pkt entry should only be allocated and posted by the progress engine.
682 * However, to mock a receive completion, we have to allocate an rx entry
683 * and modify it out of band. The proess engine grow the rx pool in the first
684 * call and set efa_rdm_ep->efa_rx_pkts_posted as the rx pool size. Here we
685 * follow the progress engine to set the efa_rx_pkts_posted counter manually
686 * TODO: modify the rx pkt as part of the ibv cq poll mock so we don't have to
687 * allocate pkt entry and hack the pkt counters.
688 */
689 pkt_entry = efa_rdm_pke_alloc(efa_rdm_ep, efa_rdm_ep->efa_rx_pkt_pool, EFA_RDM_PKE_FROM_EFA_RX_POOL);
690 assert_non_null(pkt_entry);
691 efa_rdm_ep->efa_rx_pkts_posted = efa_rdm_ep_get_rx_pool_size(efa_rdm_ep);
692
693 pkt_attr.msg_id = 0;
694 pkt_attr.connid = raw_addr.qkey;
695 /* Packet type must be in [EFA_RDM_REQ_PKT_BEGIN, EFA_RDM_EXTRA_REQ_PKT_END) */
696 efa_unit_test_eager_msgrtm_pkt_construct(pkt_entry, &pkt_attr);
697
698 /* Setup CQ */
699 ibv_cqx->wr_id = (uintptr_t)pkt_entry;
700 ibv_cqx->start_poll = &efa_mock_ibv_start_poll_return_mock;
701 ibv_cqx->next_poll = &efa_mock_ibv_next_poll_check_function_called_and_return_mock;
702 ibv_cqx->end_poll = &efa_mock_ibv_end_poll_check_mock;
703 ibv_cqx->read_slid = &efa_mock_ibv_read_slid_return_mock;
704 ibv_cqx->read_byte_len = &efa_mock_ibv_read_byte_len_return_mock;
705 ibv_cqx->read_opcode = &efa_mock_ibv_read_opcode_return_mock;
706 ibv_cqx->read_qp_num = &efa_mock_ibv_read_qp_num_return_mock;
707 ibv_cqx->read_wc_flags = &efa_mock_ibv_read_wc_flags_return_mock;
708 ibv_cqx->read_src_qp = &efa_mock_ibv_read_src_qp_return_mock;
709
710 if (support_efadv_cq) {
711 efadv_cq = efadv_cq_from_ibv_cq_ex(ibv_cqx);
712 assert_non_null(efadv_cq);
713 efadv_cq->wc_read_sgid = &efa_mock_efadv_wc_read_sgid_return_zero_code_and_expect_next_poll_and_set_gid;
714
715 /* Return unknown AH from efadv */
716 will_return(efa_mock_efadv_wc_read_sgid_return_zero_code_and_expect_next_poll_and_set_gid, raw_addr.raw);
717 } else {
718 expect_function_call(efa_mock_ibv_next_poll_check_function_called_and_return_mock);
719 }
720
721 /* Read 1 entry with unknown AH */
722 will_return(efa_mock_ibv_start_poll_return_mock, 0);
723 will_return(efa_mock_ibv_next_poll_check_function_called_and_return_mock, ENOENT);
724 will_return(efa_mock_ibv_end_poll_check_mock, NULL);
725 will_return(efa_mock_ibv_read_slid_return_mock, 0xffff); // slid=0xffff(-1) indicates an unknown AH
726 will_return(efa_mock_ibv_read_byte_len_return_mock, pkt_entry->pkt_size);
727 will_return_maybe(efa_mock_ibv_read_opcode_return_mock, IBV_WC_RECV);
728 will_return_maybe(efa_mock_ibv_read_qp_num_return_mock, efa_rdm_ep->base_ep.qp->qp_num);
729 will_return_maybe(efa_mock_ibv_read_wc_flags_return_mock, 0);
730 will_return_maybe(efa_mock_ibv_read_src_qp_return_mock, raw_addr.qpn);
731
732 /* Post receive buffer */
733 ret = fi_recv(resource->ep, recv_buff.buff, recv_buff.size, fi_mr_desc(recv_buff.mr), peer_addr, NULL /* context */);
734 assert_int_equal(ret, 0);
735
736 if (remove_peer) {
737 ret = fi_av_remove(resource->av, &peer_addr, 1, 0);
738 assert_int_equal(ret, 0);
739 }
740
741 ret = fi_cq_read(resource->cq, &cq_entry, 1);
742
743 if (remove_peer || !support_efadv_cq) {
744 /* Ignored WC because the peer is removed, or EFA device does not support extended CQ */
745 assert_int_equal(ret, -FI_EAGAIN);
746 }
747 else {
748 /* Found 1 matching rxe */
749 assert_int_equal(ret, 1);
750 }
751
752 efa_unit_test_buff_destruct(&recv_buff);
753 }
754
755 /**
756 * @brief Verify that RDM endpoint fi_cq_read recovers unknown peer AH
757 * by querying efadv to get raw address.
758 * A fake peer is registered in AV. The endpoint receives a packet from it,
759 * for which the EFA device returns an unknown AH. The endpoint will retrieve
760 * the peer's raw address using efadv verbs, and recover it's AH using
761 * Raw:QPN:QKey.
762 *
763 * @param[in] state struct efa_resource that is managed by the framework
764 */
765 void test_ibv_cq_ex_read_recover_forgotten_peer_ah(struct efa_resource **state)
766 {
767 struct efa_resource *resource = *state;
768 test_impl_ibv_cq_ex_read_unknow_peer_ah(resource, false, true);
769 }
770
771 /**
772 * @brief Verify that RDM endpoint falls back to ibv_create_cq_ex if rdma-core
773 * provides efadv_create_cq verb but EFA device does not support EFA DV CQ.
774 * In this case the endpoint will not attempt to recover a forgotten peer's address.
775 *
776 * @param[in] state struct efa_resource that is managed by the framework
777 */
778 void test_rdm_fallback_to_ibv_create_cq_ex_cq_read_ignore_forgotton_peer(struct efa_resource **state)
779 {
780 struct efa_resource *resource = *state;
781 test_impl_ibv_cq_ex_read_unknow_peer_ah(resource, false, false);
782 }
783
784 /**
785 * @brief Verify that RDM endpoint progress engine ignores unknown peer AH
786 * if the peer is not registered in AV, e.g. removed.
787 * The endpoint receives a packet from an alien peer, which corresponds to
788 * an unknown AH. The endpoint attempts to look up the AH for the peer but
789 * was rightly unable to, thus ignoring the packet.
790 *
791 * @param[in] state struct efa_resource that is managed by the framework
792 */
793 void test_ibv_cq_ex_read_ignore_removed_peer(struct efa_resource **state)
794 {
795 struct efa_resource *resource = *state;
796 test_impl_ibv_cq_ex_read_unknow_peer_ah(resource, true, true);
797 }
798 #else
799 void test_ibv_cq_ex_read_recover_forgotten_peer_ah()
800 {
801 skip();
802 }
803 void test_rdm_fallback_to_ibv_create_cq_ex_cq_read_ignore_forgotton_peer()
804 {
805 skip();
806 }
807 void test_ibv_cq_ex_read_ignore_removed_peer()
808 {
809 skip();
810 }
811 #endif
812
813 static void test_efa_cq_read_prep(struct efa_resource *resource,
814 int ibv_wc_opcode, int status, int vendor_error,
815 struct efa_context *ctx, int wc_flags,
816 bool is_unsolicited_write_recv)
817 {
818 int ret;
819 size_t raw_addr_len = sizeof(struct efa_ep_addr);
820 struct efa_ep_addr raw_addr;
821 struct ibv_cq_ex *ibv_cqx;
822 struct efa_cq *efa_cq;
823 struct efa_base_ep *base_ep;
824 fi_addr_t addr;
825
826 efa_unit_test_resource_construct(resource, FI_EP_RDM, EFA_DIRECT_FABRIC_NAME);
827
828 ret = fi_getname(&resource->ep->fid, &raw_addr, &raw_addr_len);
829 assert_int_equal(ret, 0);
830 raw_addr.qpn = 1;
831 raw_addr.qkey = 0x1234;
832 ret = fi_av_insert(resource->av, &raw_addr, 1, &addr, 0 /* flags */, NULL /* context */);
833 assert_int_equal(ret, 1);
834 if (ctx)
835 ctx->addr = addr;
836
837 base_ep = container_of(resource->ep, struct efa_base_ep, util_ep.ep_fid);
838 efa_cq = container_of(resource->cq, struct efa_cq, util_cq.cq_fid.fid);
839 ibv_cqx = efa_cq->ibv_cq.ibv_cq_ex;
840
841 /* Make wr_id as 0 for unsolicited write recv as a stress test */
842 ibv_cqx->wr_id = is_unsolicited_write_recv ? 0 : (uintptr_t) ctx;
843 ibv_cqx->status = status;
844 ibv_cqx->start_poll = &efa_mock_ibv_start_poll_return_mock;
845 ibv_cqx->next_poll = &efa_mock_ibv_next_poll_return_mock;
846 ibv_cqx->end_poll = &efa_mock_ibv_end_poll_check_mock;
847 ibv_cqx->read_opcode = &efa_mock_ibv_read_opcode_return_mock;
848 ibv_cqx->read_qp_num = &efa_mock_ibv_read_qp_num_return_mock;
849 ibv_cqx->read_wc_flags = &efa_mock_ibv_read_wc_flags_return_mock;
850 ibv_cqx->read_imm_data = &efa_mock_ibv_wc_read_imm_data_return_mock;
851 ibv_cqx->read_slid = &efa_mock_ibv_read_slid_return_mock;
852 ibv_cqx->read_src_qp = &efa_mock_ibv_read_src_qp_return_mock;
853 ibv_cqx->read_byte_len = &efa_mock_ibv_read_byte_len_return_mock;
854 ibv_cqx->read_vendor_err = &efa_mock_ibv_read_vendor_err_return_mock;
855
856 will_return(efa_mock_ibv_start_poll_return_mock, 0);
857 will_return_maybe(efa_mock_ibv_next_poll_return_mock, ENOENT);
858 will_return_maybe(efa_mock_ibv_end_poll_check_mock, NULL);
859 will_return_maybe(efa_mock_ibv_read_opcode_return_mock, ibv_wc_opcode);
860 will_return_maybe(efa_mock_ibv_read_qp_num_return_mock, base_ep->qp->qp_num);
861 will_return_maybe(efa_mock_ibv_read_vendor_err_return_mock, vendor_error);
862 will_return_maybe(efa_mock_ibv_read_byte_len_return_mock, 4096);
863 will_return_maybe(efa_mock_ibv_read_slid_return_mock, efa_av_addr_to_conn(base_ep->av, addr)->ah->ahn);
864 will_return_maybe(efa_mock_ibv_read_src_qp_return_mock, raw_addr.qpn);
865 will_return_maybe(efa_mock_ibv_read_wc_flags_return_mock, wc_flags);
866 will_return_maybe(efa_mock_ibv_wc_read_imm_data_return_mock, 0x1);
867
868 #if HAVE_CAPS_UNSOLICITED_WRITE_RECV
869 if (efa_use_unsolicited_write_recv()) {
870 efadv_cq_from_ibv_cq_ex(ibv_cqx)->wc_is_unsolicited = &efa_mock_efadv_wc_is_unsolicited;
871 will_return_maybe(efa_mock_efadv_wc_is_unsolicited, is_unsolicited_write_recv);
872 }
873 #endif
874 }
875
876 /**
877 * @brief test EFA CQ's fi_cq_read() works properly when rdma-core return
878 * success status for send operation without wr_id (inject). In this case
879 * no completion should be generated.
880 */
881 void test_efa_cq_read_no_completion(struct efa_resource **state)
882 {
883 struct efa_resource *resource = *state;
884 struct fi_cq_data_entry cq_entry;
885 int ret;
886
887 test_efa_cq_read_prep(resource, IBV_WC_SEND, IBV_WC_SUCCESS, 0,
888 NULL, 0, false);
889
890 ret = fi_cq_read(resource->cq, &cq_entry, 1);
891 assert_int_equal(ret, -FI_EAGAIN);
892 }
893
894 /**
895 * @brief test EFA CQ's fi_cq_read() works properly when rdma-core return
896 * success status for send operation.
897 */
898 void test_efa_cq_read_send_success(struct efa_resource **state)
899 {
900 struct efa_resource *resource = *state;
901 struct efa_context *efa_context;
902 struct fi_context2 ctx;
903 struct fi_cq_data_entry cq_entry = {0};
904 int ret;
905
906 efa_context = (struct efa_context *) &ctx;
907 efa_context->completion_flags = FI_SEND | FI_MSG;
908
909 test_efa_cq_read_prep(resource, IBV_WC_SEND, IBV_WC_SUCCESS, 0,
910 efa_context, 0, false);
911
912 ret = fi_cq_read(resource->cq, &cq_entry, 1);
913 assert_int_equal(ret, 1);
914
915 assert_true(efa_context == cq_entry.op_context);
916 assert_true(cq_entry.flags == efa_context->completion_flags);
917 }
918
919 /**
920 * @brief test EFA CQ's fi_cq_read() works properly when rdma-core return
921 * success status for senddata operation.
922 */
923 void test_efa_cq_read_senddata_success(struct efa_resource **state)
924 {
925 struct efa_resource *resource = *state;
926 struct efa_context *efa_context;
927 struct fi_context2 ctx;
928 struct fi_cq_data_entry cq_entry = {0};
929 int ret;
930
931 efa_context = (struct efa_context *) &ctx;
932 efa_context->completion_flags = FI_SEND | FI_MSG | FI_REMOTE_CQ_DATA;
933
934 test_efa_cq_read_prep(resource, IBV_WC_SEND, IBV_WC_SUCCESS, 0,
935 efa_context, IBV_WC_WITH_IMM, false);
936
937 ret = fi_cq_read(resource->cq, &cq_entry, 1);
938 assert_int_equal(ret, 1);
939
940 assert_true(efa_context == cq_entry.op_context);
941 assert_true(cq_entry.flags == efa_context->completion_flags);
942 }
943
944 /**
945 * @brief test EFA CQ's fi_cq_read() works properly when rdma-core return
946 * success status for recv operation.
947 */
948 void test_efa_cq_read_recv_success(struct efa_resource **state)
949 {
950 struct efa_resource *resource = *state;
951 struct efa_context *efa_context;
952 struct fi_cq_data_entry cq_entry;
953 struct fi_context2 ctx;
954 int ret;
955
956 efa_context = (struct efa_context *) &ctx;
957 efa_context->completion_flags = FI_RECV | FI_MSG;
958
959 test_efa_cq_read_prep(resource, IBV_WC_RECV, IBV_WC_SUCCESS, 0,
960 efa_context, 0, false);
961
962 ret = fi_cq_read(resource->cq, &cq_entry, 1);
963 assert_int_equal(ret, 1);
964
965 assert_true(efa_context == cq_entry.op_context);
966 assert_true(efa_context->completion_flags == cq_entry.flags);
967 }
968
969 /**
970 * @brief test EFA CQ's fi_cq_read() works properly when rdma-core return
971 * success status for senddata operation.
972 */
973 void test_efa_cq_read_write_success(struct efa_resource **state)
974 {
975 struct efa_resource *resource = *state;
976 struct efa_context *efa_context;
977 struct fi_context2 ctx;
978 struct fi_cq_data_entry cq_entry = {0};
979 int ret;
980
981 efa_context = (struct efa_context *) &ctx;
982 efa_context->completion_flags = FI_WRITE | FI_RMA;
983
984 test_efa_cq_read_prep(resource, IBV_WC_SEND, IBV_WC_SUCCESS, 0,
985 efa_context, 0, false);
986
987 ret = fi_cq_read(resource->cq, &cq_entry, 1);
988 assert_int_equal(ret, 1);
989
990 assert_true(efa_context == cq_entry.op_context);
991 assert_true(cq_entry.flags == efa_context->completion_flags);
992 }
993
994 /**
995 * @brief test EFA CQ's fi_cq_read() works properly when rdma-core return
996 * success status for writedata operation.
997 */
998 void test_efa_cq_read_writedata_success(struct efa_resource **state)
999 {
1000 struct efa_resource *resource = *state;
1001 struct efa_context *efa_context;
1002 struct fi_context2 ctx;
1003 struct fi_cq_data_entry cq_entry = {0};
1004 int ret;
1005
1006 efa_context = (struct efa_context *) &ctx;
1007 efa_context->completion_flags = FI_WRITE | FI_RMA | FI_REMOTE_CQ_DATA;
1008
1009 test_efa_cq_read_prep(resource, IBV_WC_RDMA_WRITE, IBV_WC_SUCCESS, 0,
1010 efa_context, IBV_WC_WITH_IMM, false);
1011
1012 ret = fi_cq_read(resource->cq, &cq_entry, 1);
1013 assert_int_equal(ret, 1);
1014
1015 assert_true(efa_context == cq_entry.op_context);
1016 assert_true(cq_entry.flags == efa_context->completion_flags);
1017 }
1018
1019 /**
1020 * @brief test EFA CQ's fi_cq_read() works properly when rdma-core return
1021 * success status for read operation.
1022 */
1023 void test_efa_cq_read_read_success(struct efa_resource **state)
1024 {
1025 struct efa_resource *resource = *state;
1026 struct efa_context *efa_context;
1027 struct fi_context2 ctx;
1028 struct fi_cq_data_entry cq_entry = {0};
1029 int ret;
1030
1031 efa_context = (struct efa_context *) &ctx;
1032 efa_context->completion_flags = FI_READ | FI_RMA;
1033
1034 test_efa_cq_read_prep(resource, IBV_WC_RDMA_READ, IBV_WC_SUCCESS, 0,
1035 efa_context, 0, false);
1036
1037 ret = fi_cq_read(resource->cq, &cq_entry, 1);
1038 assert_int_equal(ret, 1);
1039
1040 assert_true(efa_context == cq_entry.op_context);
1041 assert_true(cq_entry.flags == efa_context->completion_flags);
1042 }
1043
1044 /**
1045 * @brief test EFA CQ's fi_cq_read() works properly when rdma-core return
1046 * success status for recv rdma with imm operation.
1047 */
1048 void test_efa_cq_read_recv_rdma_with_imm_success(struct efa_resource **state)
1049 {
1050 struct efa_resource *resource = *state;
1051 struct fi_cq_data_entry cq_entry;
1052 int ret;
1053
1054 test_efa_cq_read_prep(resource, IBV_WC_RECV_RDMA_WITH_IMM, IBV_WC_SUCCESS, 0,
1055 NULL, IBV_WC_WITH_IMM, true);
1056
1057 ret = fi_cq_read(resource->cq, &cq_entry, 1);
1058 assert_int_equal(ret, 1);
1059
1060 assert_true(cq_entry.op_context == NULL);
1061 assert_true(cq_entry.flags == (FI_REMOTE_WRITE | FI_REMOTE_CQ_DATA | FI_RMA));
1062 }
1063
1064 static void efa_cq_check_cq_err_entry(struct efa_resource *resource, int vendor_error) {
1065 struct fi_cq_err_entry cq_err_entry = {0};
1066 const char *strerror;
1067 int ret;
1068
1069 /* Allocate memory to read CQ error */
1070 cq_err_entry.err_data_size = EFA_ERROR_MSG_BUFFER_LENGTH;
1071 cq_err_entry.err_data = malloc(cq_err_entry.err_data_size);
1072 assert_non_null(cq_err_entry.err_data);
1073
1074 ret = fi_cq_readerr(resource->cq, &cq_err_entry, 0);
1075 assert_true(cq_err_entry.err_data_size > 0);
1076 strerror = fi_cq_strerror(resource->cq, cq_err_entry.prov_errno,
1077 cq_err_entry.err_data, NULL, 0);
1078
1079 assert_int_equal(ret, 1);
1080 assert_int_not_equal(cq_err_entry.err, FI_SUCCESS);
1081 assert_int_equal(cq_err_entry.prov_errno, vendor_error);
1082 assert_true(strlen(strerror) > 0);
1083 }
1084
1085 /**
1086 * @brief test EFA CQ's fi_cq_read()/fi_cq_readerr() works properly when rdma-core return bad status for send.
1087 *
1088 * When the send operation failed, fi_cq_read() should return -FI_EAVAIL, which means error available.
1089 * then user should call fi_cq_readerr() to get an error CQ entry that contain error code.
1090 *
1091 * @param[in] state struct efa_resource that is managed by the framework
1092 */
1093 void test_efa_cq_read_send_failure(struct efa_resource **state)
1094 {
1095 struct efa_resource *resource = *state;
1096 struct efa_context *efa_context;
1097 struct fi_cq_data_entry cq_entry;
1098 struct fi_context2 ctx;
1099 int ret;
1100
1101 efa_context = (struct efa_context *) &ctx;
1102 efa_context->completion_flags = FI_SEND | FI_MSG;
1103
1104 test_efa_cq_read_prep(resource, IBV_WC_SEND, IBV_WC_GENERAL_ERR,
1105 EFA_IO_COMP_STATUS_LOCAL_ERROR_UNRESP_REMOTE, (struct efa_context *) &ctx, 0, false);
1106
1107 ret = fi_cq_read(resource->cq, &cq_entry, 1);
1108 assert_int_equal(ret, -FI_EAVAIL);
1109
1110 efa_cq_check_cq_err_entry(resource,
1111 EFA_IO_COMP_STATUS_LOCAL_ERROR_UNRESP_REMOTE);
1112 }
1113
1114 /**
1115 * @brief test EFA CQ's fi_cq_read()/fi_cq_readerr() works properly when rdma-core return bad status for recv.
1116 *
1117 * When the recv operation failed, fi_cq_read() should return -FI_EAVAIL, which means error available.
1118 * then user should call fi_cq_readerr() to get an error CQ entry that contain error code.
1119 *
1120 * @param[in] state struct efa_resource that is managed by the framework
1121 */
1122 void test_efa_cq_read_recv_failure(struct efa_resource **state)
1123 {
1124 struct efa_resource *resource = *state;
1125 struct efa_context *efa_context;
1126 struct fi_cq_data_entry cq_entry;
1127 struct fi_context2 ctx;
1128 int ret;
1129
1130 efa_context = (struct efa_context *) &ctx;
1131 efa_context->completion_flags = FI_RECV | FI_MSG;
1132
1133 test_efa_cq_read_prep(resource, IBV_WC_RECV, IBV_WC_GENERAL_ERR,
1134 EFA_IO_COMP_STATUS_LOCAL_ERROR_UNRESP_REMOTE, (struct efa_context *) &ctx, 0, false);
1135
1136 ret = fi_cq_read(resource->cq, &cq_entry, 1);
1137 assert_int_equal(ret, -FI_EAVAIL);
1138
1139 efa_cq_check_cq_err_entry(resource,
1140 EFA_IO_COMP_STATUS_LOCAL_ERROR_UNRESP_REMOTE);
1141 }
1142
1143 /**
1144 * @brief test EFA CQ's fi_cq_read()/fi_cq_readerr() works properly when rdma-core return bad status for recv.
1145 *
1146 * When the recv_rdma_with_imm operation failed, fi_cq_read() should return -FI_EAVAIL, which means error available.
1147 * then user should call fi_cq_readerr() to get an error CQ entry that contain error code.
1148 *
1149 * @param[in] state struct efa_resource that is managed by the framework
1150 */
1151 void test_efa_cq_recv_rdma_with_imm_failure(struct efa_resource **state)
1152 {
1153 struct efa_resource *resource = *state;
1154 struct fi_cq_data_entry cq_entry;
1155 int ret;
1156
1157 test_efa_cq_read_prep(resource, IBV_WC_RECV_RDMA_WITH_IMM, IBV_WC_GENERAL_ERR,
1158 EFA_IO_COMP_STATUS_LOCAL_ERROR_UNRESP_REMOTE, NULL, IBV_WC_WITH_IMM, true);
1159
1160 ret = fi_cq_read(resource->cq, &cq_entry, 1);
1161 assert_int_equal(ret, -FI_EAVAIL);
1162
1163 efa_cq_check_cq_err_entry(resource,
1164 EFA_IO_COMP_STATUS_LOCAL_ERROR_UNRESP_REMOTE);
1165 }
1166