Debian Codegraph @

src/deps/miniasync/extras/dml/data_mover_dml.c

raw
1 // SPDX-License-Identifier: BSD-3-Clause
2 /* Copyright 2021-2022, Intel Corporation */
3
4 #include <dml/dml.h>
5 #include <libminiasync.h>
6 #include <stdbool.h>
7 #include <stdlib.h>
8
9 #include "core/membuf.h"
10 #include "core/out.h"
11 #include "libminiasync-vdm-dml.h"
12
13 #define SUPPORTED_FLAGS VDM_F_MEM_DURABLE | VDM_F_NO_CACHE_HINT
14 /*
15 * XXX: This flag should be defined in DML header but for some reason isn't
16 * this flag is needed to guarantee that writes to persistent memory
17 * are persistent at the time DSA operation is completed
18 */
19 #define DSA_F_DESTINATION_READBACK (1 < 14)
20
21 struct data_mover_dml {
22 struct vdm base; /* must be first */
23 dml_path_t path;
24 struct membuf *membuf;
25 };
26
27 /*
28 * data_mover_dml_translate_flags -- translate miniasync-vdm-dml flags
29 */
30 static void
31 data_mover_dml_translate_flags(uint64_t flags, uint64_t *dml_flags)
32 {
33 ASSERTeq((flags & ~VDM_F_VALID_FLAGS), 0);
34
35 *dml_flags = DML_FLAG_PREFETCH_CACHE;
36 for (uint64_t iflag = 1; flags > 0; iflag = iflag << 1) {
37 if ((flags & iflag) == 0)
38 continue;
39
40 switch (iflag) {
41 /*
42 * write to destination is identified as write to
43 * durable memory
44 */
45 case VDM_F_MEM_DURABLE:
46 *dml_flags |= DML_FLAG_DST1_DURABLE;
47 *dml_flags |= DSA_F_DESTINATION_READBACK;
48 break;
49 case VDM_F_NO_CACHE_HINT:
50 *dml_flags &= ~DML_FLAG_PREFETCH_CACHE;
51 break;
52 default: /* shouldn't be possible */
53 ASSERT(0);
54 }
55
56 /* remove translated flag from the flags to be translated */
57 flags = flags & (~iflag);
58 }
59 }
60
61 /*
62 * data_mover_dml_memcpy_job_init -- initializes new memcpy dml job
63 */
64 static dml_job_t *
65 data_mover_dml_memcpy_job_init(dml_job_t *dml_job,
66 void *dest, void *src, size_t n, uint64_t flags)
67 {
68 uint64_t dml_flags = 0;
69 data_mover_dml_translate_flags(flags, &dml_flags);
70
71 dml_job->operation = DML_OP_MEM_MOVE;
72 dml_job->source_first_ptr = (uint8_t *)src;
73 dml_job->destination_first_ptr = (uint8_t *)dest;
74 dml_job->source_length = n;
75 dml_job->destination_length = n;
76 dml_job->flags = DML_FLAG_COPY_ONLY | dml_flags;
77
78 return dml_job;
79 }
80
81 /*
82 * data_mover_dml_memmove_job_init -- initializes new memmove dml job
83 */
84 static dml_job_t *
85 data_mover_dml_memmove_job_init(dml_job_t *dml_job,
86 void *dest, void *src, size_t n, uint64_t flags)
87 {
88 uint64_t dml_flags = 0;
89 data_mover_dml_translate_flags(flags, &dml_flags);
90
91 dml_job->operation = DML_OP_MEM_MOVE;
92 dml_job->source_first_ptr = (uint8_t *)src;
93 dml_job->destination_first_ptr = (uint8_t *)dest;
94 dml_job->source_length = n;
95 dml_job->destination_length = n;
96 dml_job->flags = dml_flags;
97
98 return dml_job;
99 }
100
101 /*
102 * data_mover_dml_memset_job_init -- initializes new memset dml job
103 */
104 static dml_job_t *
105 data_mover_dml_memset_job_init(dml_job_t *dml_job,
106 void *ptr, int value, size_t n, uint64_t flags)
107 {
108 uint64_t dml_flags = 0;
109 data_mover_dml_translate_flags(flags, &dml_flags);
110
111 dml_job->operation = DML_OP_FILL;
112 dml_job->destination_first_ptr = (uint8_t *)ptr;
113 dml_job->destination_length = n;
114 dml_job->flags = dml_flags;
115
116 /*
117 * Original 'memset' implementation converts the provided 'value' into
118 * a 'unsigned char' type.
119 */
120 unsigned char c = (unsigned char)value;
121
122 /* Populate the pattern */
123 for (size_t i = 0; i < 8 && i < n; i++) {
124 dml_job->pattern[i] = (uint8_t)c;
125 }
126
127 return dml_job;
128 }
129
130 /*
131 * data_mover_dml_flush_job_init -- initializes new flush dml job
132 */
133 static dml_job_t *
134 data_mover_dml_flush_job_init(dml_job_t *dml_job,
135 void *dest, size_t n, uint64_t flags)
136 {
137 uint64_t dml_flags = 0;
138 data_mover_dml_translate_flags(flags, &dml_flags);
139
140 dml_job->operation = DML_OP_CACHE_FLUSH;
141 dml_job->destination_first_ptr = (uint8_t *)dest;
142 dml_job->destination_length = n;
143 dml_job->flags = dml_flags;
144
145 return dml_job;
146 }
147
148 /*
149 * data_mover_dml_job_delete -- delete job struct
150 */
151 static void
152 data_mover_dml_job_delete(dml_job_t **dml_job)
153 {
154 dml_finalize_job(*dml_job);
155 }
156
157 /*
158 * data_mover_dml_memory_op_job_submit -- submit job for memory operations,
159 * which include memcpy and memmove (nonblocking)
160 */
161 static void *
162 data_mover_dml_memory_op_job_submit(dml_job_t *dml_job)
163 {
164 dml_status_t status;
165 status = dml_submit_job(dml_job);
166
167 return status == DML_STATUS_OK ? dml_job->destination_first_ptr : NULL;
168 }
169
170 /*
171 * data_mover_dml_operation_new -- create a new DML job
172 */
173 static void *
174 data_mover_dml_operation_new(struct vdm *vdm,
175 const enum vdm_operation_type type)
176 {
177 struct data_mover_dml *vdm_dml = (struct data_mover_dml *)vdm;
178 dml_status_t status;
179 uint32_t job_size;
180 dml_job_t *dml_job;
181
182 switch (type) {
183 case VDM_OPERATION_MEMCPY:
184 case VDM_OPERATION_MEMMOVE:
185 case VDM_OPERATION_MEMSET:
186 case VDM_OPERATION_FLUSH:
187 break;
188 default:
189 ASSERT(0); /* unreachable */
190 }
191
192 status = dml_get_job_size(vdm_dml->path, &job_size);
193 if (status != DML_STATUS_OK)
194 return NULL;
195
196 dml_job = membuf_alloc(vdm_dml->membuf, job_size);
197 if (dml_job == NULL)
198 return NULL;
199
200 dml_status_t job_status =
201 dml_init_job(vdm_dml->path, dml_job);
202 if (job_status != DML_STATUS_OK) {
203 membuf_free(dml_job);
204 return NULL;
205 }
206
207 return dml_job;
208 }
209
210 /*
211 * data_mover_dml_operation_delete -- delete a DML job
212 */
213 static void
214 data_mover_dml_operation_delete(void *data,
215 const struct vdm_operation *operation,
216 struct vdm_operation_output *output)
217 {
218 dml_job_t *job = (dml_job_t *)data;
219 dml_status_t status = dml_check_job(job);
220 switch (status) {
221 case DML_STATUS_BEING_PROCESSED:
222 ASSERT(0 && "dml job being deleted during processing");
223 case DML_STATUS_JOB_CORRUPTED:
224 output->result = VDM_ERROR_JOB_CORRUPTED;
225 break;
226 case DML_STATUS_OK:
227 output->result = VDM_SUCCESS;
228 break;
229 default:
230 ASSERT(0);
231 }
232
233 switch (job->operation) {
234 case DML_OP_MEM_MOVE:
235 if (job->flags & DML_FLAG_COPY_ONLY) {
236 output->type = VDM_OPERATION_MEMCPY;
237 output->output.memcpy.dest =
238 job->destination_first_ptr;
239 } else {
240 output->type = VDM_OPERATION_MEMMOVE;
241 output->output.memmove.dest =
242 job->destination_first_ptr;
243 }
244 break;
245 case DML_OP_FILL:
246 output->type = VDM_OPERATION_MEMSET;
247 output->output.memset.str = job->destination_first_ptr;
248 break;
249 case DML_OP_CACHE_FLUSH:
250 output->type = VDM_OPERATION_FLUSH;
251 break;
252 default:
253 ASSERT(0);
254 }
255
256 data_mover_dml_job_delete(&job);
257
258 membuf_free(data);
259 }
260
261 /*
262 * data_mover_dml_operation_check -- checks the status of a DML job
263 */
264 enum future_state
265 data_mover_dml_operation_check(void *data,
266 const struct vdm_operation *operation)
267 {
268 SUPPRESS_UNUSED(operation);
269
270 dml_job_t *job = (dml_job_t *)data;
271
272 dml_status_t status = dml_check_job(job);
273 switch (status) {
274 case DML_STATUS_BEING_PROCESSED:
275 return FUTURE_STATE_RUNNING;
276 case DML_STATUS_JOB_CORRUPTED:
277 case DML_STATUS_OK:
278 return FUTURE_STATE_COMPLETE;
279 default:
280 ASSERT(0);
281 }
282 }
283
284 /*
285 * data_mover_dml_operation_start -- start ('submit') asynchronous dml job
286 */
287 int
288 data_mover_dml_operation_start(void *data,
289 const struct vdm_operation *operation, struct future_notifier *n)
290 {
291 if (n) {
292 n->notifier_used = FUTURE_NOTIFIER_NONE;
293 }
294
295 dml_job_t *job = (dml_job_t *)data;
296
297 switch (operation->type) {
298 case VDM_OPERATION_MEMCPY:
299 data_mover_dml_memcpy_job_init(job,
300 operation->data.memcpy.dest,
301 operation->data.memcpy.src,
302 operation->data.memcpy.n,
303 operation->data.memcpy.flags);
304 data_mover_dml_memory_op_job_submit(job);
305 break;
306 case VDM_OPERATION_MEMMOVE:
307 data_mover_dml_memmove_job_init(job,
308 operation->data.memmove.dest,
309 operation->data.memmove.src,
310 operation->data.memmove.n,
311 operation->data.memmove.flags);
312 data_mover_dml_memory_op_job_submit(job);
313 break;
314 case VDM_OPERATION_MEMSET:
315 data_mover_dml_memset_job_init(job,
316 operation->data.memset.str,
317 operation->data.memset.c,
318 operation->data.memset.n,
319 operation->data.memset.flags);
320 data_mover_dml_memory_op_job_submit(job);
321 break;
322 case VDM_OPERATION_FLUSH:
323 data_mover_dml_flush_job_init(job,
324 operation->data.flush.dest,
325 operation->data.flush.n,
326 operation->data.flush.flags);
327 data_mover_dml_memory_op_job_submit(job);
328 break;
329 default:
330 ASSERT(0);
331 }
332
333 return 0;
334 }
335
336 int
337 has_property_dmd(void *fut, enum future_property property)
338 {
339 if (property == FUTURE_PROPERTY_ASYNC)
340 return 1;
341 return 0;
342 }
343
344 /*
345 * data_mover_dml_vdm -- dml asynchronous memcpy
346 */
347 static struct vdm data_mover_dml_vdm = {
348 .op_new = data_mover_dml_operation_new,
349 .op_delete = data_mover_dml_operation_delete,
350 .op_check = data_mover_dml_operation_check,
351 .op_start = data_mover_dml_operation_start,
352 .capabilities = SUPPORTED_FLAGS,
353 .has_property = has_property_dmd,
354 };
355
356 /*
357 * data_mover_dml_new -- creates a new dml-based data mover instance
358 */
359 struct data_mover_dml *
360 data_mover_dml_new(enum data_mover_dml_type type)
361 {
362 struct data_mover_dml *vdm_dml = malloc(sizeof(struct data_mover_dml));
363 if (vdm_dml == NULL)
364 return NULL;
365
366 vdm_dml->membuf = membuf_new(vdm_dml);
367 vdm_dml->base = data_mover_dml_vdm;
368 switch (type) {
369 case DATA_MOVER_DML_HARDWARE:
370 vdm_dml->path = DML_PATH_HW;
371 break;
372 case DATA_MOVER_DML_SOFTWARE:
373 vdm_dml->path = DML_PATH_SW;
374 break;
375 case DATA_MOVER_DML_AUTO:
376 vdm_dml->path = DML_PATH_AUTO;
377 break;
378 default:
379 ASSERT(0);
380 }
381
382 return vdm_dml;
383 }
384
385 /*
386 * data_mover_dml_get_vdm -- returns the vdm for dml data mover
387 */
388 struct vdm *
389 data_mover_dml_get_vdm(struct data_mover_dml *dmd)
390 {
391 return &dmd->base;
392 }
393
394 /*
395 * data_mover_dml_delete -- deletes a vdm_dml instance
396 */
397 void
398 data_mover_dml_delete(struct data_mover_dml *dmd)
399 {
400 membuf_delete(dmd->membuf);
401 free(dmd);
402 }