Debian Codegraph @

prov/efa/test/efa_unit_test_rma.c

raw
1 /* SPDX-License-Identifier: BSD-2-Clause OR GPL-2.0-only */
2 /* SPDX-FileCopyrightText: Copyright Amazon.com, Inc. or its affiliates. All
3 * rights reserved. */
4
5 #include "efa_unit_tests.h"
6 #include "ofi_util.h"
7
8 extern struct fi_ops_rma efa_rma_ops;
9
10 static void test_efa_rma_prep(struct efa_resource *resource, fi_addr_t *addr)
11 {
12 struct ibv_qp_ex *ibv_qpx;
13 struct efa_ep_addr raw_addr;
14 struct efa_base_ep *base_ep;
15 size_t raw_addr_len = sizeof(raw_addr);
16 int ret;
17
18 efa_unit_test_resource_construct(resource, FI_EP_RDM, EFA_DIRECT_FABRIC_NAME);
19
20 base_ep = container_of(resource->ep, struct efa_base_ep, util_ep.ep_fid);
21 /* Add rma caps explicitly to ep->info to allow local test */
22 base_ep->info->caps |= FI_RMA;
23 ibv_qpx = base_ep->qp->ibv_qp_ex;
24 ibv_qpx->wr_start = &efa_mock_ibv_wr_start_no_op;
25 /* this mock will save the send work request (wr) in a global list */
26 ibv_qpx->wr_rdma_read = &efa_mock_ibv_wr_rdma_read_save_wr;
27 ibv_qpx->wr_rdma_write = &efa_mock_ibv_wr_rdma_write_save_wr;
28 ibv_qpx->wr_rdma_write_imm = &efa_mock_ibv_wr_rdma_write_imm_save_wr;
29 ibv_qpx->wr_set_sge_list = &efa_mock_ibv_wr_set_sge_list_no_op;
30 ibv_qpx->wr_set_ud_addr = &efa_mock_ibv_wr_set_ud_addr_no_op;
31 ibv_qpx->wr_complete = &efa_mock_ibv_wr_complete_no_op;
32
33 ret = fi_getname(&resource->ep->fid, &raw_addr, &raw_addr_len);
34 assert_int_equal(ret, 0);
35 raw_addr.qpn = 1;
36 raw_addr.qkey = 0x1234;
37 ret = fi_av_insert(resource->av, &raw_addr, 1, addr, 0 /* flags */,
38 NULL /* context */);
39 assert_int_equal(ret, 1);
40 }
41
42 void test_efa_rma_read(struct efa_resource **state)
43 {
44 struct efa_resource *resource = *state;
45 struct efa_unit_test_buff local_buff;
46 fi_addr_t src_addr;
47 void *desc;
48 int ret;
49 uint64_t remote_addr = 0x87654321;
50 uint64_t remote_key = 123456;
51
52 test_efa_rma_prep(resource, &src_addr);
53 efa_unit_test_buff_construct(&local_buff, resource, 4096 /* buff_size */);
54
55 desc = fi_mr_desc(local_buff.mr);
56
57 assert_int_equal(g_ibv_submitted_wr_id_cnt, 0);
58 ret = fi_read(resource->ep, local_buff.buff, local_buff.size, desc,
59 src_addr, remote_addr, remote_key, NULL /* context */);
60 assert_int_equal(ret, 0);
61 assert_int_equal(g_ibv_submitted_wr_id_cnt, 1);
62
63 efa_unit_test_buff_destruct(&local_buff);
64 }
65
66 void test_efa_rma_readv(struct efa_resource **state)
67 {
68 struct efa_resource *resource = *state;
69 struct efa_unit_test_buff local_buff;
70 struct iovec iov;
71 fi_addr_t src_addr;
72 void *desc;
73 int ret;
74 uint64_t remote_addr = 0x87654321;
75 uint64_t remote_key = 123456;
76
77 test_efa_rma_prep(resource, &src_addr);
78 efa_unit_test_buff_construct(&local_buff, resource, 4096 /* buff_size */);
79
80 iov.iov_base = local_buff.buff;
81 iov.iov_len = local_buff.size;
82 desc = fi_mr_desc(local_buff.mr);
83
84 assert_int_equal(g_ibv_submitted_wr_id_cnt, 0);
85 ret = fi_readv(resource->ep, &iov, &desc, 1, src_addr, remote_addr,
86 remote_key, NULL /* context */);
87 assert_int_equal(ret, 0);
88 assert_int_equal(g_ibv_submitted_wr_id_cnt, 1);
89
90 efa_unit_test_buff_destruct(&local_buff);
91 }
92
93 void test_efa_rma_readmsg(struct efa_resource **state)
94 {
95 struct efa_resource *resource = *state;
96 struct efa_unit_test_buff local_buff;
97 struct iovec iov;
98 struct fi_msg_rma msg = {0};
99 struct fi_rma_iov rma_iov;
100 fi_addr_t src_addr;
101 void *desc;
102 int ret;
103
104 test_efa_rma_prep(resource, &src_addr);
105 efa_unit_test_buff_construct(&local_buff, resource, 4096 /* buff_size */);
106
107 iov.iov_base = local_buff.buff;
108 iov.iov_len = local_buff.size;
109 desc = fi_mr_desc(local_buff.mr);
110 rma_iov.len = local_buff.size;
111 rma_iov.addr = 0x87654321;
112 rma_iov.key = 123456;
113 efa_unit_test_construct_msg_rma(&msg, &iov, &desc, 1, src_addr,
114 &rma_iov, 1, NULL, 0);
115
116 assert_int_equal(g_ibv_submitted_wr_id_cnt, 0);
117 ret = fi_readmsg(resource->ep, &msg, 0);
118 assert_int_equal(ret, 0);
119 assert_int_equal(g_ibv_submitted_wr_id_cnt, 1);
120
121 efa_unit_test_buff_destruct(&local_buff);
122 }
123
124 void test_efa_rma_write(struct efa_resource **state)
125 {
126 struct efa_resource *resource = *state;
127 struct efa_unit_test_buff local_buff;
128 fi_addr_t dest_addr;
129 void *desc;
130 int ret;
131 uint64_t remote_addr = 0x87654321;
132 uint64_t remote_key = 123456;
133
134 test_efa_rma_prep(resource, &dest_addr);
135 efa_unit_test_buff_construct(&local_buff, resource, 4096 /* buff_size */);
136
137 desc = fi_mr_desc(local_buff.mr);
138
139 assert_int_equal(g_ibv_submitted_wr_id_cnt, 0);
140 ret = fi_write(resource->ep, local_buff.buff, local_buff.size, desc,
141 dest_addr, remote_addr, remote_key, NULL /* context */);
142 assert_int_equal(ret, 0);
143 assert_int_equal(g_ibv_submitted_wr_id_cnt, 1);
144
145 efa_unit_test_buff_destruct(&local_buff);
146 }
147
148 void test_efa_rma_writev(struct efa_resource **state)
149 {
150 struct efa_resource *resource = *state;
151 struct efa_unit_test_buff local_buff;
152 struct iovec iov;
153 fi_addr_t dest_addr;
154 void *desc;
155 int ret;
156 uint64_t remote_addr = 0x87654321;
157 uint64_t remote_key = 123456;
158
159 test_efa_rma_prep(resource, &dest_addr);
160 efa_unit_test_buff_construct(&local_buff, resource, 4096 /* buff_size */);
161
162 iov.iov_base = local_buff.buff;
163 iov.iov_len = local_buff.size;
164 desc = fi_mr_desc(local_buff.mr);
165
166 assert_int_equal(g_ibv_submitted_wr_id_cnt, 0);
167 ret = fi_writev(resource->ep, &iov, &desc, 1, dest_addr, remote_addr,
168 remote_key, NULL /* context */);
169 assert_int_equal(ret, 0);
170 assert_int_equal(g_ibv_submitted_wr_id_cnt, 1);
171
172 efa_unit_test_buff_destruct(&local_buff);
173 }
174
175 void test_efa_rma_writemsg(struct efa_resource **state)
176 {
177 struct efa_resource *resource = *state;
178 struct efa_unit_test_buff local_buff;
179 struct iovec iov;
180 struct fi_msg_rma msg = {0};
181 struct fi_rma_iov rma_iov;
182 fi_addr_t dest_addr;
183 void *desc;
184 int ret;
185
186 test_efa_rma_prep(resource, &dest_addr);
187 efa_unit_test_buff_construct(&local_buff, resource, 4096 /* buff_size */);
188
189 iov.iov_base = local_buff.buff;
190 iov.iov_len = local_buff.size;
191 desc = fi_mr_desc(local_buff.mr);
192 rma_iov.len = local_buff.size;
193 rma_iov.addr = 0x87654321;
194 rma_iov.key = 123456;
195 efa_unit_test_construct_msg_rma(&msg, &iov, &desc, 1, dest_addr, &rma_iov,
196 1, NULL, 0);
197
198 assert_int_equal(g_ibv_submitted_wr_id_cnt, 0);
199 ret = fi_writemsg(resource->ep, &msg, 0);
200 assert_int_equal(ret, 0);
201 assert_int_equal(g_ibv_submitted_wr_id_cnt, 1);
202
203 efa_unit_test_buff_destruct(&local_buff);
204 }
205
206 void test_efa_rma_writedata(struct efa_resource **state)
207 {
208 struct efa_resource *resource = *state;
209 struct efa_unit_test_buff local_buff;
210 fi_addr_t dest_addr;
211 void *desc;
212 int ret;
213 uint64_t remote_addr = 0x87654321;
214 uint64_t remote_key = 123456;
215
216 test_efa_rma_prep(resource, &dest_addr);
217 efa_unit_test_buff_construct(&local_buff, resource, 4096 /* buff_size */);
218
219 desc = fi_mr_desc(local_buff.mr);
220
221 assert_int_equal(g_ibv_submitted_wr_id_cnt, 0);
222 ret = fi_writedata(resource->ep, local_buff.buff, local_buff.size, desc,
223 0, dest_addr, remote_addr, remote_key,
224 NULL /* context */);
225 assert_int_equal(ret, 0);
226 assert_int_equal(g_ibv_submitted_wr_id_cnt, 1);
227
228 efa_unit_test_buff_destruct(&local_buff);
229 }
230
231 void test_efa_rma_inject_write(struct efa_resource **state)
232 {
233 struct efa_resource *resource = *state;
234 struct efa_unit_test_buff local_buff;
235 fi_addr_t dest_addr;
236 int ret;
237 uint64_t remote_addr = 0x87654321;
238 uint64_t remote_key = 123456;
239
240 test_efa_rma_prep(resource, &dest_addr);
241 efa_unit_test_buff_construct(&local_buff, resource, 32 /* buff_size */);
242
243 ret = fi_inject_write(resource->ep, local_buff.buff, local_buff.size,
244 dest_addr, remote_addr, remote_key);
245 assert_int_equal(ret, -FI_ENOSYS);
246
247 efa_unit_test_buff_destruct(&local_buff);
248 }
249
250 void test_efa_rma_inject_writedata(struct efa_resource **state)
251 {
252 struct efa_resource *resource = *state;
253 struct efa_unit_test_buff local_buff;
254 fi_addr_t dest_addr;
255 int ret;
256 uint64_t remote_addr = 0x87654321;
257 uint64_t remote_key = 123456;
258
259 test_efa_rma_prep(resource, &dest_addr);
260 efa_unit_test_buff_construct(&local_buff, resource, 32 /* buff_size */);
261
262 ret = fi_inject_writedata(resource->ep, local_buff.buff,
263 local_buff.size, 0, dest_addr, remote_addr,
264 remote_key);
265 assert_int_equal(ret, -FI_ENOSYS);
266
267 efa_unit_test_buff_destruct(&local_buff);
268 }
269
270 void test_efa_rma_writemsg_with_inject(struct efa_resource **state)
271 {
272 struct efa_resource *resource = *state;
273 struct efa_unit_test_buff local_buff;
274 struct iovec iov;
275 struct fi_msg_rma msg = {0};
276 struct fi_rma_iov rma_iov;
277 fi_addr_t dest_addr;
278 void *desc;
279 int ret;
280
281 test_efa_rma_prep(resource, &dest_addr);
282 efa_unit_test_buff_construct(&local_buff, resource, 4096 /* buff_size */);
283
284 iov.iov_base = local_buff.buff;
285 iov.iov_len = local_buff.size;
286 desc = fi_mr_desc(local_buff.mr);
287 rma_iov.len = local_buff.size;
288 rma_iov.addr = 0x87654321;
289 rma_iov.key = 123456;
290 efa_unit_test_construct_msg_rma(&msg, &iov, &desc, 1, dest_addr, &rma_iov,
291 1, NULL, 0);
292
293 ret = fi_writemsg(resource->ep, &msg, FI_INJECT);
294 assert_int_equal(ret, -FI_ENOSYS);
295
296 efa_unit_test_buff_destruct(&local_buff);
297 }