Debian Codegraph @

prov/efa/test/efa_unit_test_mr.c

raw
1 /* SPDX-License-Identifier: BSD-2-Clause OR GPL-2.0-only */
2 /* SPDX-FileCopyrightText: Copyright Amazon.com, Inc. or its affiliates. All rights reserved. */
3
4 #include "efa_unit_tests.h"
5
6 static void test_efa_mr_impl(struct efa_domain *efa_domain, struct fid_mr *mr,
7 int mr_reg_count, int mr_reg_size, bool gdrcopy_flag)
8 {
9 struct efa_mr *efa_mr;
10
11 assert_int_equal(efa_domain->ibv_mr_reg_ct, mr_reg_count);
12 assert_int_equal(efa_domain->ibv_mr_reg_sz, mr_reg_size);
13
14 if (mr) {
15 efa_mr = container_of(mr, struct efa_mr, mr_fid);
16 if (cuda_is_gdrcopy_enabled()) {
17 if (gdrcopy_flag)
18 assert_true(efa_mr->peer.flags &
19 OFI_HMEM_DATA_DEV_REG_HANDLE);
20 else
21 assert_false(efa_mr->peer.flags &
22 OFI_HMEM_DATA_DEV_REG_HANDLE);
23 }
24 }
25 }
26
27 void test_efa_rdm_mr_reg_host_memory(struct efa_resource **state)
28 {
29 struct efa_resource *resource = *state;
30 struct efa_domain *efa_domain;
31 size_t mr_size = 64;
32 void *buf;
33 struct fid_mr *mr = NULL;
34
35 efa_unit_test_resource_construct(resource, FI_EP_RDM, EFA_FABRIC_NAME);
36
37 efa_domain = container_of(resource->domain, struct efa_domain,
38 util_domain.domain_fid);
39
40 buf = malloc(mr_size);
41 assert_non_null(buf);
42
43 test_efa_mr_impl(efa_domain, mr, 0, 0, false);
44
45 assert_int_equal(fi_mr_reg(resource->domain, buf, mr_size,
46 FI_SEND | FI_RECV, 0, 0, 0, &mr, NULL),
47 0);
48
49 /* No GDRCopy registration for host memory */
50 test_efa_mr_impl(efa_domain, mr, 1, mr_size, false);
51
52 assert_int_equal(fi_close(&mr->fid), 0);
53 test_efa_mr_impl(efa_domain, NULL, 0, 0, false);
54 free(buf);
55 }
56
57 #if HAVE_CUDA
58 void test_efa_rdm_mr_reg_cuda_memory(struct efa_resource **state)
59 {
60 struct efa_resource *resource = *state;
61 struct efa_domain *efa_domain;
62 size_t mr_size = 64;
63 void *buf;
64 struct fid_mr *mr = NULL;
65 struct fi_mr_attr mr_reg_attr = { 0 };
66 struct iovec iovec;
67 int err;
68
69 if (hmem_ops[FI_HMEM_CUDA].initialized &&
70 g_efa_hmem_info[FI_HMEM_CUDA].p2p_supported_by_device) {
71 resource->hints = efa_unit_test_alloc_hints_hmem(
72 FI_EP_RDM, EFA_FABRIC_NAME);
73 efa_unit_test_resource_construct_with_hints(resource, FI_EP_RDM,
74 FI_VERSION(2, 0),
75 resource->hints,
76 true, true);
77
78 efa_domain = container_of(resource->domain, struct efa_domain,
79 util_domain.domain_fid);
80 test_efa_mr_impl(efa_domain, mr, 0, 0, false);
81
82 err = ofi_cudaMalloc(&buf, mr_size);
83 assert_int_equal(err, 0);
84 assert_non_null(buf);
85
86 mr_reg_attr.access = FI_SEND | FI_RECV;
87 mr_reg_attr.iface = FI_HMEM_CUDA;
88 iovec.iov_base = buf;
89 iovec.iov_len = mr_size;
90 mr_reg_attr.mr_iov = &iovec;
91 mr_reg_attr.iov_count = 1;
92
93 err = fi_mr_regattr(resource->domain, &mr_reg_attr, 0, &mr);
94 assert_int_equal(err, 0);
95
96 /* FI_MR_DMABUF flag was not set, so GDRCopy should be registered if available */
97 test_efa_mr_impl(efa_domain, mr, 1, mr_size, true);
98
99 assert_int_equal(fi_close(&mr->fid), 0);
100 test_efa_mr_impl(efa_domain, NULL, 0, 0, false);
101
102 err = ofi_cudaFree(buf);
103 assert_int_equal(err, 0);
104 }
105 }
106 #else
107 void test_efa_rdm_mr_reg_cuda_memory(struct efa_resource **state)
108 {
109 skip();
110 }
111 #endif
112
113 #if HAVE_CUDA
114 void test_efa_direct_mr_reg_no_gdrcopy(struct efa_resource **state)
115 {
116 struct efa_resource *resource = *state;
117 struct efa_domain *efa_domain;
118 size_t mr_size = 64;
119 void *buf;
120 struct fid_mr *mr = NULL;
121 struct fi_mr_attr mr_reg_attr = { 0 };
122 struct iovec iovec;
123 int err;
124
125 if (g_efa_hmem_info[FI_HMEM_CUDA].initialized &&
126 g_efa_hmem_info[FI_HMEM_CUDA].p2p_supported_by_device) {
127 resource->hints = efa_unit_test_alloc_hints_hmem(
128 FI_EP_RDM, EFA_DIRECT_FABRIC_NAME);
129 efa_unit_test_resource_construct_with_hints(resource, FI_EP_RDM,
130 FI_VERSION(2, 0),
131 resource->hints,
132 true, true);
133
134 efa_domain = container_of(resource->domain, struct efa_domain,
135 util_domain.domain_fid);
136 test_efa_mr_impl(efa_domain, mr, 0, 0, false);
137
138 err = ofi_cudaMalloc(&buf, mr_size);
139 assert_int_equal(err, 0);
140 assert_non_null(buf);
141
142 mr_reg_attr.access = FI_SEND | FI_RECV;
143 mr_reg_attr.iface = FI_HMEM_CUDA;
144 iovec.iov_base = buf;
145 iovec.iov_len = mr_size;
146 mr_reg_attr.mr_iov = &iovec;
147 mr_reg_attr.iov_count = 1;
148
149 err = fi_mr_regattr(resource->domain, &mr_reg_attr, 0, &mr);
150 assert_int_equal(err, 0);
151
152 /* no GDRCopy in the efa-direct path */
153 test_efa_mr_impl(efa_domain, mr, 1, mr_size, false);
154
155 assert_int_equal(fi_close(&mr->fid), 0);
156 test_efa_mr_impl(efa_domain, NULL, 0, 0, false);
157
158 err = ofi_cudaFree(buf);
159 assert_int_equal(err, 0);
160 }
161 }
162 #else
163 void test_efa_direct_mr_reg_no_gdrcopy(struct efa_resource **state)
164 {
165 skip();
166 }
167 #endif