/* * Copyright (C) by Argonne National Laboratory * See COPYRIGHT in top-level directory */ #include "mpiimpl.h" #include "tsp_impl.h" #include "gentran_impl.h" /* UT_icd helper structure for vtx_t utarray */ UT_icd vtx_t_icd = { sizeof(MPII_Genutil_vtx_t), NULL, MPII_Genutil_vtx_copy, MPII_Genutil_vtx_dtor }; /* UT_icd helper structure for MPII_Genutil_vtx_type_t utarray */ UT_icd vtx_type_t_icd = { sizeof(MPII_Genutil_vtx_type_t), NULL, NULL, NULL }; int MPIR_TSP_sched_create(MPIR_TSP_sched_t * s, bool is_persistent) { MPII_Genutil_sched_t *sched = MPL_malloc(sizeof(MPII_Genutil_sched_t), MPL_MEM_COLL); MPIR_Assert(sched != NULL); sched->total_vtcs = 0; sched->completed_vtcs = 0; sched->last_fence = -1; /* initialize array for storing vertices */ utarray_init(&sched->vtcs, &vtx_t_icd); utarray_init(&sched->buffers, &ut_ptr_icd); utarray_init(&sched->generic_types, &vtx_type_t_icd); sched->issued_head = NULL; sched->issued_tail = NULL; sched->is_persistent = is_persistent; *s = sched; return MPI_SUCCESS; } int MPIR_TSP_sched_free(MPIR_TSP_sched_t s) { MPII_Genutil_sched_t *sched = s; int mpi_errno ATTRIBUTE((unused)) = MPI_SUCCESS; int i; vtx_t *vtx; void **p; /* free up the sched resources */ vtx = ut_type_array(&sched->vtcs, vtx_t *); MPII_Genutil_vtx_type_t *vtype = ut_type_array(&sched->generic_types, MPII_Genutil_vtx_type_t *); for (i = 0; i < sched->total_vtcs; i++, vtx++) { MPIR_ERR_CHKANDJUMP(vtx == NULL, mpi_errno, MPI_ERR_OTHER, "**nullvertex"); switch (vtx->vtx_kind) { case MPII_GENUTIL_VTX_KIND__ISEND: MPIR_Comm_release(vtx->u.isend.comm); MPIR_Datatype_release_if_not_builtin(vtx->u.isend.dt); break; case MPII_GENUTIL_VTX_KIND__IRECV: MPIR_Comm_release(vtx->u.irecv.comm); MPIR_Datatype_release_if_not_builtin(vtx->u.irecv.dt); break; case MPII_GENUTIL_VTX_KIND__IRECV_STATUS: MPIR_Comm_release(vtx->u.irecv_status.comm); MPIR_Datatype_release_if_not_builtin(vtx->u.irecv_status.dt); break; case MPII_GENUTIL_VTX_KIND__IMCAST: MPIR_Comm_release(vtx->u.imcast.comm); MPIR_Datatype_release_if_not_builtin(vtx->u.imcast.dt); MPL_free(vtx->u.imcast.req); utarray_done(&vtx->u.imcast.dests); break; case MPII_GENUTIL_VTX_KIND__REDUCE_LOCAL: MPIR_Op_release_if_not_builtin(vtx->u.reduce_local.op); MPIR_Datatype_release_if_not_builtin(vtx->u.reduce_local.datatype); break; case MPII_GENUTIL_VTX_KIND__LOCALCOPY: MPIR_Datatype_release_if_not_builtin(vtx->u.localcopy.sendtype); MPIR_Datatype_release_if_not_builtin(vtx->u.localcopy.recvtype); break; case MPII_GENUTIL_VTX_KIND__SCHED: /* In normal case, sub schedule is free'ed when it is done * only when the sub-scheduler is persistent */ if (vtx->u.sched.is_persistent) { mpi_errno = MPIR_TSP_sched_free(vtx->u.sched.sched); MPIR_ERR_CHECK(mpi_errno); } break; default: if (vtx->vtx_kind > MPII_GENUTIL_VTX_KIND__LAST) { MPII_Genutil_vtx_type_t *type = vtype + vtx->vtx_kind - MPII_GENUTIL_VTX_KIND__LAST - 1; MPIR_Assert(type != NULL); if (type->free_fn != NULL) { mpi_errno = type->free_fn(vtx->u.generic.data); MPIR_ERR_CHECK(mpi_errno); } } } } /* free up the allocated buffers */ p = NULL; while ((p = (void **) utarray_next(&sched->buffers, p))) MPL_free(*p); utarray_done(&sched->vtcs); utarray_done(&sched->buffers); utarray_done(&sched->generic_types); MPL_free(sched); fn_exit: return mpi_errno; fn_fail: goto fn_exit; } int MPIR_TSP_sched_new_type(MPIR_TSP_sched_t s, MPIR_TSP_sched_issue_fn issue_fn, MPIR_TSP_sched_complete_fn complete_fn, MPIR_TSP_sched_free_fn free_fn) { MPII_Genutil_sched_t *sched = s; MPII_Genutil_vtx_type_t *vtype, *newtype; int type_id, i; /* only free_fn can be NULL */ MPIR_Assert(issue_fn && complete_fn); /* search if the new type already exists in the table */ vtype = ut_type_array(&sched->generic_types, MPII_Genutil_vtx_type_t *); for (i = 0; i < utarray_len(&sched->generic_types); i++) { if (vtype->issue_fn == issue_fn && vtype->complete_fn == complete_fn && vtype->free_fn == free_fn) { return MPII_GENUTIL_VTX_KIND__LAST + i + 1; } vtype++; } type_id = MPII_GENUTIL_VTX_KIND__LAST + utarray_len(&sched->generic_types) + 1; utarray_extend_back(&sched->generic_types, MPL_MEM_COLL); newtype = (MPII_Genutil_vtx_type_t *) utarray_back(&sched->generic_types); newtype->id = type_id; newtype->issue_fn = issue_fn; newtype->complete_fn = complete_fn; newtype->free_fn = free_fn; return type_id; } int MPIR_TSP_sched_generic(int type_id, void *data, MPIR_TSP_sched_t s, int n_in_vtcs, int *in_vtcs, int *vtx_id) { MPII_Genutil_sched_t *sched = s; int mpi_errno = MPI_SUCCESS; vtx_t *vtxp; if (type_id <= MPII_GENUTIL_VTX_KIND__LAST || type_id - MPII_GENUTIL_VTX_KIND__LAST > utarray_len(&sched->generic_types)) { MPIR_ERR_SETANDJUMP(mpi_errno, MPI_ERR_OTHER, "**nomem"); } MPIR_Assert(vtx_id); *vtx_id = MPII_Genutil_vtx_create(sched, &vtxp); vtxp->vtx_kind = type_id; vtxp->u.generic.data = data; MPII_Genutil_vtx_add_dependencies(sched, *vtx_id, n_in_vtcs, in_vtcs); MPL_DBG_MSG_FMT(MPIR_DBG_COLL, VERBOSE, (MPL_DBG_FDEST, "Gentran: schedule [%d] generic [%d]", *vtx_id, type_id)); fn_exit: return mpi_errno; fn_fail: goto fn_exit; } int MPIR_TSP_sched_isend(const void *buf, MPI_Aint count, MPI_Datatype dt, int dest, int tag, MPIR_Comm * comm_ptr, MPIR_TSP_sched_t s, int n_in_vtcs, int *in_vtcs, int *vtx_id) { MPII_Genutil_sched_t *sched = s; vtx_t *vtxp; int mpi_errno = MPI_SUCCESS; /* assign a new vertex */ *vtx_id = MPII_Genutil_vtx_create(sched, &vtxp); vtxp->vtx_kind = MPII_GENUTIL_VTX_KIND__ISEND; MPII_Genutil_vtx_add_dependencies(sched, *vtx_id, n_in_vtcs, in_vtcs); /* store the arguments */ vtxp->u.isend.buf = buf; vtxp->u.isend.count = count; vtxp->u.isend.dt = dt; vtxp->u.isend.dest = dest; vtxp->u.isend.tag = tag; vtxp->u.isend.comm = comm_ptr; /* the user may free the comm & type after initiating but before the * underlying send is actually posted, so we must add a reference here and * release it at entry completion time */ MPIR_Comm_add_ref(comm_ptr); MPIR_Datatype_add_ref_if_not_builtin(dt); MPL_DBG_MSG_FMT(MPIR_DBG_COLL, VERBOSE, (MPL_DBG_FDEST, "Gentran: schedule [%d] isend", *vtx_id)); return mpi_errno; } int MPIR_TSP_sched_irecv(void *buf, MPI_Aint count, MPI_Datatype dt, int source, int tag, MPIR_Comm * comm_ptr, MPIR_TSP_sched_t s, int n_in_vtcs, int *in_vtcs, int *vtx_id) { MPII_Genutil_sched_t *sched = s; vtx_t *vtxp; int mpi_errno = MPI_SUCCESS; /* assign a new vertex */ *vtx_id = MPII_Genutil_vtx_create(sched, &vtxp); vtxp->vtx_kind = MPII_GENUTIL_VTX_KIND__IRECV; MPII_Genutil_vtx_add_dependencies(sched, *vtx_id, n_in_vtcs, in_vtcs); /* record the arguments */ vtxp->u.irecv.buf = buf; vtxp->u.irecv.count = count; vtxp->u.irecv.dt = dt; vtxp->u.irecv.src = source; vtxp->u.irecv.tag = tag; vtxp->u.irecv.comm = comm_ptr; MPIR_Comm_add_ref(comm_ptr); MPIR_Datatype_add_ref_if_not_builtin(dt); MPL_DBG_MSG_FMT(MPIR_DBG_COLL, VERBOSE, (MPL_DBG_FDEST, "Gentran: schedule [%d] irecv", *vtx_id)); return mpi_errno; } int MPIR_TSP_sched_irecv_status(void *buf, MPI_Aint count, MPI_Datatype dt, int source, int tag, MPIR_Comm * comm_ptr, MPI_Status * status, MPIR_TSP_sched_t sched, int n_in_vtcs, int *in_vtcs, int *vtx_id) { vtx_t *vtxp; int mpi_errno = MPI_SUCCESS;; /* assign a new vertex */ *vtx_id = MPII_Genutil_vtx_create(sched, &vtxp); vtxp->vtx_kind = MPII_GENUTIL_VTX_KIND__IRECV_STATUS; MPII_Genutil_vtx_add_dependencies(sched, *vtx_id, n_in_vtcs, in_vtcs); /* record the arguments */ vtxp->u.irecv_status.buf = buf; vtxp->u.irecv_status.count = count; vtxp->u.irecv_status.dt = dt; vtxp->u.irecv_status.src = source; vtxp->u.irecv_status.tag = tag; vtxp->u.irecv_status.comm = comm_ptr; vtxp->u.irecv_status.status = status; MPIR_Comm_add_ref(comm_ptr); MPIR_Datatype_add_ref_if_not_builtin(dt); MPL_DBG_MSG_FMT(MPIR_DBG_COLL, VERBOSE, (MPL_DBG_FDEST, "Gentran: schedule [%d] irecv_status", *vtx_id)); return mpi_errno; } int MPIR_TSP_sched_imcast(const void *buf, MPI_Aint count, MPI_Datatype dt, int *dests, int num_dests, int tag, MPIR_Comm * comm_ptr, MPIR_TSP_sched_t s, int n_in_vtcs, int *in_vtcs, int *vtx_id) { MPII_Genutil_sched_t *sched = s; vtx_t *vtxp; int mpi_errno = MPI_SUCCESS; /* assign a new vertex */ *vtx_id = MPII_Genutil_vtx_create(sched, &vtxp); vtxp->vtx_kind = MPII_GENUTIL_VTX_KIND__IMCAST; MPII_Genutil_vtx_add_dependencies(sched, *vtx_id, n_in_vtcs, in_vtcs); /* store the arguments */ vtxp->u.imcast.buf = (void *) buf; vtxp->u.imcast.count = count; vtxp->u.imcast.dt = dt; vtxp->u.imcast.num_dests = num_dests; utarray_init(&vtxp->u.imcast.dests, &ut_int_icd); utarray_reserve(&vtxp->u.imcast.dests, num_dests, MPL_MEM_COLL); memcpy(ut_int_array(&vtxp->u.imcast.dests), dests, num_dests * sizeof(int)); vtxp->u.imcast.tag = tag; vtxp->u.imcast.comm = comm_ptr; vtxp->u.imcast.req = (struct MPIR_Request **) MPL_malloc(sizeof(struct MPIR_Request *) * num_dests, MPL_MEM_COLL); vtxp->u.imcast.last_complete = -1; MPIR_Comm_add_ref(comm_ptr); MPIR_Datatype_add_ref_if_not_builtin(dt); MPL_DBG_MSG_FMT(MPIR_DBG_COLL, VERBOSE, (MPL_DBG_FDEST, "Gentran: schedule [%d] imcast", *vtx_id)); return mpi_errno; } int MPIR_TSP_sched_reduce_local(const void *inbuf, void *inoutbuf, MPI_Aint count, MPI_Datatype datatype, MPI_Op op, MPIR_TSP_sched_t s, int n_in_vtcs, int *in_vtcs, int *vtx_id) { MPII_Genutil_sched_t *sched = s; vtx_t *vtxp; int mpi_errno = MPI_SUCCESS; /* assign a new vertex */ *vtx_id = MPII_Genutil_vtx_create(sched, &vtxp); vtxp->vtx_kind = MPII_GENUTIL_VTX_KIND__REDUCE_LOCAL; MPII_Genutil_vtx_add_dependencies(sched, *vtx_id, n_in_vtcs, in_vtcs); /* record the arguments */ vtxp->u.reduce_local.inbuf = inbuf; vtxp->u.reduce_local.inoutbuf = inoutbuf; vtxp->u.reduce_local.count = count; vtxp->u.reduce_local.datatype = datatype; vtxp->u.reduce_local.op = op; MPIR_Datatype_add_ref_if_not_builtin(datatype); MPIR_Op_add_ref_if_not_builtin(op); MPL_DBG_MSG_FMT(MPIR_DBG_COLL, VERBOSE, (MPL_DBG_FDEST, "Gentran: schedule [%d] reduce_local", *vtx_id)); return mpi_errno; } int MPIR_TSP_sched_localcopy(const void *sendbuf, MPI_Aint sendcount, MPI_Datatype sendtype, void *recvbuf, MPI_Aint recvcount, MPI_Datatype recvtype, MPIR_TSP_sched_t s, int n_in_vtcs, int *in_vtcs, int *vtx_id) { MPII_Genutil_sched_t *sched = s; vtx_t *vtxp; int mpi_errno = MPI_SUCCESS; /* assign a new vertex */ *vtx_id = MPII_Genutil_vtx_create(sched, &vtxp); vtxp->vtx_kind = MPII_GENUTIL_VTX_KIND__LOCALCOPY; MPII_Genutil_vtx_add_dependencies(sched, *vtx_id, n_in_vtcs, in_vtcs); /* record the arguments */ vtxp->u.localcopy.sendbuf = sendbuf; vtxp->u.localcopy.sendcount = sendcount; vtxp->u.localcopy.sendtype = sendtype; vtxp->u.localcopy.recvbuf = recvbuf; vtxp->u.localcopy.recvcount = recvcount; vtxp->u.localcopy.recvtype = recvtype; MPIR_Datatype_add_ref_if_not_builtin(sendtype); MPIR_Datatype_add_ref_if_not_builtin(recvtype); MPL_DBG_MSG_FMT(MPIR_DBG_COLL, VERBOSE, (MPL_DBG_FDEST, "Gentran: schedule [%d] localcopy", *vtx_id)); return mpi_errno; } /* Transport function that adds a callback type vertex in the graph */ int MPIR_TSP_sched_cb(MPIR_TSP_cb_t cb_p, void *cb_data, MPIR_TSP_sched_t sched, int n_in_vtcs, int *in_vtcs, int *vtx_id) { vtx_t *vtxp; int mpi_errno = MPI_SUCCESS; /* assign a new vertex */ *vtx_id = MPII_Genutil_vtx_create(sched, &vtxp); vtxp->vtx_kind = MPII_GENUTIL_VTX_KIND__CB; MPII_Genutil_vtx_add_dependencies(sched, *vtx_id, n_in_vtcs, in_vtcs); /* record the arguments */ vtxp->u.cb.cb_p = cb_p; vtxp->u.cb.cb_data = cb_data; MPL_DBG_MSG_FMT(MPIR_DBG_COLL, VERBOSE, (MPL_DBG_FDEST, "Gentran: schedule [%d] cb", *vtx_id)); return mpi_errno; } /* Transport function that adds a no op vertex in the graph that has * all the vertices posted before it as incoming vertices */ int MPIR_TSP_sched_sink(MPIR_TSP_sched_t s, int *vtx_id) { MPII_Genutil_sched_t *sched = s; MPL_DBG_MSG_FMT(MPIR_DBG_COLL, VERBOSE, (MPL_DBG_FDEST, "Gentran: sched [sink] total=%d", sched->total_vtcs)); vtx_t *vtxp; vtx_t *sched_fence; int i, n_in_vtcs = 0; int *in_vtcs; int mpi_errno = MPI_SUCCESS; MPIR_CHKLMEM_DECL(); /* assign a new vertex */ *vtx_id = MPII_Genutil_vtx_create(sched, &vtxp); vtxp->vtx_kind = MPII_GENUTIL_VTX_KIND__SINK; MPIR_CHKLMEM_MALLOC(in_vtcs, sizeof(int) * (*vtx_id)); /* record incoming vertices */ sched_fence = ut_type_array(&sched->vtcs, vtx_t *) + *vtx_id - 1; MPIR_ERR_CHKANDJUMP(sched_fence == NULL, mpi_errno, MPI_ERR_OTHER, "**nofence"); for (i = *vtx_id - 1; i >= 0; i--) { if (sched_fence->vtx_kind == MPII_GENUTIL_VTX_KIND__FENCE) /* no need to record this and any vertex before fence. * Dependency on the last fence call will be added by * the subsequent call to MPIC_Genutil_vtx_add_dependencies function */ break; else { /* this vertex only depends on the sink when it has no * outgoing dependence */ if (utarray_len(&sched_fence->out_vtcs) == 0) in_vtcs[n_in_vtcs++] = i; } sched_fence--; } MPII_Genutil_vtx_add_dependencies(sched, *vtx_id, n_in_vtcs, in_vtcs); MPIR_CHKLMEM_FREEALL(); fn_exit: return mpi_errno; fn_fail: /* TODO - Replace this with real error handling */ MPIR_Assert(MPI_SUCCESS == mpi_errno); goto fn_exit; } int MPIR_TSP_sched_fence(MPIR_TSP_sched_t s) { MPII_Genutil_sched_t *sched = s; int mpi_errno ATTRIBUTE((unused)) = MPI_SUCCESS; int fence_id; MPL_DBG_MSG_FMT(MPIR_DBG_COLL, VERBOSE, (MPL_DBG_FDEST, "Gentran: scheduling a fence")); /* fence operation is an extension to fence, so we can reuse the fence call */ mpi_errno = MPIR_TSP_sched_sink(sched, &fence_id); MPIR_ERR_CHECK(mpi_errno); /* change the vertex kind from SINK to FENCE */ vtx_t *sched_fence = (vtx_t *) utarray_eltptr(&sched->vtcs, fence_id); MPIR_ERR_CHKANDJUMP(sched_fence == NULL, mpi_errno, MPI_ERR_OTHER, "**nofence"); sched_fence->vtx_kind = MPII_GENUTIL_VTX_KIND__FENCE; sched->last_fence = fence_id; fn_exit: return mpi_errno; fn_fail: goto fn_exit; } int MPIR_TSP_sched_selective_sink(MPIR_TSP_sched_t s, int n_in_vtcs, int *in_vtcs, int *vtx_id) { MPII_Genutil_sched_t *sched = s; vtx_t *vtxp; int mpi_errno = MPI_SUCCESS; /* assign a new vertex */ *vtx_id = MPII_Genutil_vtx_create(sched, &vtxp); vtxp->vtx_kind = MPII_GENUTIL_VTX_KIND__SELECTIVE_SINK; MPII_Genutil_vtx_add_dependencies(sched, *vtx_id, n_in_vtcs, in_vtcs); MPL_DBG_MSG_FMT(MPIR_DBG_COLL, VERBOSE, (MPL_DBG_FDEST, "Gentran: schedule [%d] selective_sink task", *vtx_id)); return mpi_errno; } void *MPIR_TSP_sched_malloc(size_t size, MPIR_TSP_sched_t s) { MPII_Genutil_sched_t *sched = s; void *addr = MPL_malloc(size, MPL_MEM_COLL); utarray_push_back(&sched->buffers, &addr, MPL_MEM_COLL); return addr; } int MPIR_TSP_sched_sub_sched(MPIR_TSP_sched_t s, MPIR_TSP_sched_t subs, int n_in_vtcs, int *in_vtcs, int *vtx_id) { MPII_Genutil_sched_t *sched = s; MPII_Genutil_sched_t *subsched = subs; int mpi_errno = MPI_SUCCESS; vtx_t *vtxp; /* assign a new vertex */ *vtx_id = MPII_Genutil_vtx_create(sched, &vtxp); vtxp->vtx_kind = MPII_GENUTIL_VTX_KIND__SCHED; MPII_Genutil_vtx_add_dependencies(sched, *vtx_id, n_in_vtcs, in_vtcs); vtxp->u.sched.sched = subsched; vtxp->u.sched.req = NULL; vtxp->u.sched.is_persistent = subsched->is_persistent; MPL_DBG_MSG_FMT(MPIR_DBG_COLL, VERBOSE, (MPL_DBG_FDEST, "Gentran: schedule [%d] sub-schedule", *vtx_id)); return mpi_errno; } int MPIR_TSP_sched_start(MPIR_TSP_sched_t s, MPIR_Comm * comm, MPIR_Request ** req) { MPII_Genutil_sched_t *sched = s; int mpi_errno = MPI_SUCCESS; int is_complete; int made_progress; MPIR_Request *reqp; MPIR_FUNC_ENTER; if (unlikely(sched->total_vtcs == 0)) { if (!sched->is_persistent) { mpi_errno = MPIR_TSP_sched_free(sched); MPIR_ERR_CHECK(mpi_errno); } *req = MPIR_Request_create_complete(MPIR_REQUEST_KIND__COLL); MPIR_ERR_CHKANDSTMT(*req == NULL, mpi_errno, MPIX_ERR_NOREQ, goto fn_fail, "**nomemreq"); goto fn_exit; } /* Create a request */ reqp = MPID_Request_create_from_comm(MPIR_REQUEST_KIND__COLL, comm); if (!reqp) MPIR_ERR_SETANDJUMP(mpi_errno, MPI_ERR_OTHER, "**nomem"); *req = reqp; MPIR_Request_add_ref(reqp); sched->req = reqp; MPIR_Assert(sched->completed_vtcs == 0); /* Kick start progress on this collective's schedule */ mpi_errno = MPII_Genutil_sched_poke(sched, &is_complete, &made_progress); if (is_complete) { MPIR_Request_complete(reqp); goto fn_exit; } /* Enqueue schedule and activate progress hook if not already activated */ reqp->u.nbc.coll.sched = (void *) sched; MPID_THREAD_CS_ENTER(VCI, MPIDIU_THREAD_TSP_QUEUE_MUTEX); DL_APPEND(MPII_coll_queue.head, &(reqp->u.nbc.coll)); MPID_THREAD_CS_EXIT(VCI, MPIDIU_THREAD_TSP_QUEUE_MUTEX); MPIR_Progress_hook_activate(MPII_Genutil_progress_hook_id); fn_exit: MPIR_FUNC_EXIT; return mpi_errno; fn_fail: goto fn_exit; } int MPIR_TSP_sched_reset(MPIR_TSP_sched_t s) { MPII_Genutil_sched_t *sched = s; int mpi_errno = MPI_SUCCESS; vtx_t *vtx; int i; MPIR_FUNC_ENTER; MPIR_Assert(sched->is_persistent); sched->completed_vtcs = 0; sched->issued_head = sched->issued_tail = NULL; vtx = ut_type_array(&sched->vtcs, vtx_t *); for (i = 0; i < sched->total_vtcs; i++, vtx++) { MPIR_ERR_CHKANDJUMP(!vtx, mpi_errno, MPI_ERR_OTHER, "**nomem"); vtx->pending_dependencies = vtx->num_dependencies; vtx->vtx_state = MPII_GENUTIL_VTX_STATE__INIT; switch (vtx->vtx_kind) { case MPII_GENUTIL_VTX_KIND__IMCAST: vtx->u.imcast.last_complete = -1; break; case MPII_GENUTIL_VTX_KIND__SCHED: MPIR_TSP_sched_reset(vtx->u.sched.sched); break; default: break; } } fn_exit: MPIR_FUNC_EXIT; return mpi_errno; fn_fail: goto fn_exit; }