// SPDX-License-Identifier: BSD-3-Clause /* Copyright 2021-2022, Intel Corporation */ #include #include #include #include #include "core/membuf.h" #include "core/out.h" #include "libminiasync-vdm-dml.h" #define SUPPORTED_FLAGS VDM_F_MEM_DURABLE | VDM_F_NO_CACHE_HINT /* * XXX: This flag should be defined in DML header but for some reason isn't * this flag is needed to guarantee that writes to persistent memory * are persistent at the time DSA operation is completed */ #define DSA_F_DESTINATION_READBACK (1 < 14) struct data_mover_dml { struct vdm base; /* must be first */ dml_path_t path; struct membuf *membuf; }; /* * data_mover_dml_translate_flags -- translate miniasync-vdm-dml flags */ static void data_mover_dml_translate_flags(uint64_t flags, uint64_t *dml_flags) { ASSERTeq((flags & ~VDM_F_VALID_FLAGS), 0); *dml_flags = DML_FLAG_PREFETCH_CACHE; for (uint64_t iflag = 1; flags > 0; iflag = iflag << 1) { if ((flags & iflag) == 0) continue; switch (iflag) { /* * write to destination is identified as write to * durable memory */ case VDM_F_MEM_DURABLE: *dml_flags |= DML_FLAG_DST1_DURABLE; *dml_flags |= DSA_F_DESTINATION_READBACK; break; case VDM_F_NO_CACHE_HINT: *dml_flags &= ~DML_FLAG_PREFETCH_CACHE; break; default: /* shouldn't be possible */ ASSERT(0); } /* remove translated flag from the flags to be translated */ flags = flags & (~iflag); } } /* * data_mover_dml_memcpy_job_init -- initializes new memcpy dml job */ static dml_job_t * data_mover_dml_memcpy_job_init(dml_job_t *dml_job, void *dest, void *src, size_t n, uint64_t flags) { uint64_t dml_flags = 0; data_mover_dml_translate_flags(flags, &dml_flags); dml_job->operation = DML_OP_MEM_MOVE; dml_job->source_first_ptr = (uint8_t *)src; dml_job->destination_first_ptr = (uint8_t *)dest; dml_job->source_length = n; dml_job->destination_length = n; dml_job->flags = DML_FLAG_COPY_ONLY | dml_flags; return dml_job; } /* * data_mover_dml_memmove_job_init -- initializes new memmove dml job */ static dml_job_t * data_mover_dml_memmove_job_init(dml_job_t *dml_job, void *dest, void *src, size_t n, uint64_t flags) { uint64_t dml_flags = 0; data_mover_dml_translate_flags(flags, &dml_flags); dml_job->operation = DML_OP_MEM_MOVE; dml_job->source_first_ptr = (uint8_t *)src; dml_job->destination_first_ptr = (uint8_t *)dest; dml_job->source_length = n; dml_job->destination_length = n; dml_job->flags = dml_flags; return dml_job; } /* * data_mover_dml_memset_job_init -- initializes new memset dml job */ static dml_job_t * data_mover_dml_memset_job_init(dml_job_t *dml_job, void *ptr, int value, size_t n, uint64_t flags) { uint64_t dml_flags = 0; data_mover_dml_translate_flags(flags, &dml_flags); dml_job->operation = DML_OP_FILL; dml_job->destination_first_ptr = (uint8_t *)ptr; dml_job->destination_length = n; dml_job->flags = dml_flags; /* * Original 'memset' implementation converts the provided 'value' into * a 'unsigned char' type. */ unsigned char c = (unsigned char)value; /* Populate the pattern */ for (size_t i = 0; i < 8 && i < n; i++) { dml_job->pattern[i] = (uint8_t)c; } return dml_job; } /* * data_mover_dml_flush_job_init -- initializes new flush dml job */ static dml_job_t * data_mover_dml_flush_job_init(dml_job_t *dml_job, void *dest, size_t n, uint64_t flags) { uint64_t dml_flags = 0; data_mover_dml_translate_flags(flags, &dml_flags); dml_job->operation = DML_OP_CACHE_FLUSH; dml_job->destination_first_ptr = (uint8_t *)dest; dml_job->destination_length = n; dml_job->flags = dml_flags; return dml_job; } /* * data_mover_dml_job_delete -- delete job struct */ static void data_mover_dml_job_delete(dml_job_t **dml_job) { dml_finalize_job(*dml_job); } /* * data_mover_dml_memory_op_job_submit -- submit job for memory operations, * which include memcpy and memmove (nonblocking) */ static void * data_mover_dml_memory_op_job_submit(dml_job_t *dml_job) { dml_status_t status; status = dml_submit_job(dml_job); return status == DML_STATUS_OK ? dml_job->destination_first_ptr : NULL; } /* * data_mover_dml_operation_new -- create a new DML job */ static void * data_mover_dml_operation_new(struct vdm *vdm, const enum vdm_operation_type type) { struct data_mover_dml *vdm_dml = (struct data_mover_dml *)vdm; dml_status_t status; uint32_t job_size; dml_job_t *dml_job; switch (type) { case VDM_OPERATION_MEMCPY: case VDM_OPERATION_MEMMOVE: case VDM_OPERATION_MEMSET: case VDM_OPERATION_FLUSH: break; default: ASSERT(0); /* unreachable */ } status = dml_get_job_size(vdm_dml->path, &job_size); if (status != DML_STATUS_OK) return NULL; dml_job = membuf_alloc(vdm_dml->membuf, job_size); if (dml_job == NULL) return NULL; dml_status_t job_status = dml_init_job(vdm_dml->path, dml_job); if (job_status != DML_STATUS_OK) { membuf_free(dml_job); return NULL; } return dml_job; } /* * data_mover_dml_operation_delete -- delete a DML job */ static void data_mover_dml_operation_delete(void *data, const struct vdm_operation *operation, struct vdm_operation_output *output) { dml_job_t *job = (dml_job_t *)data; dml_status_t status = dml_check_job(job); switch (status) { case DML_STATUS_BEING_PROCESSED: ASSERT(0 && "dml job being deleted during processing"); case DML_STATUS_JOB_CORRUPTED: output->result = VDM_ERROR_JOB_CORRUPTED; break; case DML_STATUS_OK: output->result = VDM_SUCCESS; break; default: ASSERT(0); } switch (job->operation) { case DML_OP_MEM_MOVE: if (job->flags & DML_FLAG_COPY_ONLY) { output->type = VDM_OPERATION_MEMCPY; output->output.memcpy.dest = job->destination_first_ptr; } else { output->type = VDM_OPERATION_MEMMOVE; output->output.memmove.dest = job->destination_first_ptr; } break; case DML_OP_FILL: output->type = VDM_OPERATION_MEMSET; output->output.memset.str = job->destination_first_ptr; break; case DML_OP_CACHE_FLUSH: output->type = VDM_OPERATION_FLUSH; break; default: ASSERT(0); } data_mover_dml_job_delete(&job); membuf_free(data); } /* * data_mover_dml_operation_check -- checks the status of a DML job */ enum future_state data_mover_dml_operation_check(void *data, const struct vdm_operation *operation) { SUPPRESS_UNUSED(operation); dml_job_t *job = (dml_job_t *)data; dml_status_t status = dml_check_job(job); switch (status) { case DML_STATUS_BEING_PROCESSED: return FUTURE_STATE_RUNNING; case DML_STATUS_JOB_CORRUPTED: case DML_STATUS_OK: return FUTURE_STATE_COMPLETE; default: ASSERT(0); } } /* * data_mover_dml_operation_start -- start ('submit') asynchronous dml job */ int data_mover_dml_operation_start(void *data, const struct vdm_operation *operation, struct future_notifier *n) { if (n) { n->notifier_used = FUTURE_NOTIFIER_NONE; } dml_job_t *job = (dml_job_t *)data; switch (operation->type) { case VDM_OPERATION_MEMCPY: data_mover_dml_memcpy_job_init(job, operation->data.memcpy.dest, operation->data.memcpy.src, operation->data.memcpy.n, operation->data.memcpy.flags); data_mover_dml_memory_op_job_submit(job); break; case VDM_OPERATION_MEMMOVE: data_mover_dml_memmove_job_init(job, operation->data.memmove.dest, operation->data.memmove.src, operation->data.memmove.n, operation->data.memmove.flags); data_mover_dml_memory_op_job_submit(job); break; case VDM_OPERATION_MEMSET: data_mover_dml_memset_job_init(job, operation->data.memset.str, operation->data.memset.c, operation->data.memset.n, operation->data.memset.flags); data_mover_dml_memory_op_job_submit(job); break; case VDM_OPERATION_FLUSH: data_mover_dml_flush_job_init(job, operation->data.flush.dest, operation->data.flush.n, operation->data.flush.flags); data_mover_dml_memory_op_job_submit(job); break; default: ASSERT(0); } return 0; } int has_property_dmd(void *fut, enum future_property property) { if (property == FUTURE_PROPERTY_ASYNC) return 1; return 0; } /* * data_mover_dml_vdm -- dml asynchronous memcpy */ static struct vdm data_mover_dml_vdm = { .op_new = data_mover_dml_operation_new, .op_delete = data_mover_dml_operation_delete, .op_check = data_mover_dml_operation_check, .op_start = data_mover_dml_operation_start, .capabilities = SUPPORTED_FLAGS, .has_property = has_property_dmd, }; /* * data_mover_dml_new -- creates a new dml-based data mover instance */ struct data_mover_dml * data_mover_dml_new(enum data_mover_dml_type type) { struct data_mover_dml *vdm_dml = malloc(sizeof(struct data_mover_dml)); if (vdm_dml == NULL) return NULL; vdm_dml->membuf = membuf_new(vdm_dml); vdm_dml->base = data_mover_dml_vdm; switch (type) { case DATA_MOVER_DML_HARDWARE: vdm_dml->path = DML_PATH_HW; break; case DATA_MOVER_DML_SOFTWARE: vdm_dml->path = DML_PATH_SW; break; case DATA_MOVER_DML_AUTO: vdm_dml->path = DML_PATH_AUTO; break; default: ASSERT(0); } return vdm_dml; } /* * data_mover_dml_get_vdm -- returns the vdm for dml data mover */ struct vdm * data_mover_dml_get_vdm(struct data_mover_dml *dmd) { return &dmd->base; } /* * data_mover_dml_delete -- deletes a vdm_dml instance */ void data_mover_dml_delete(struct data_mover_dml *dmd) { membuf_delete(dmd->membuf); free(dmd); }