#if HAVE_CONFIG_H # include "config.h" #endif /* $Id$ */ #if HAVE_STDIO_H # include #endif #if HAVE_STDLIB_H # include #endif #if HAVE_UNISTD_H # include #elif HAVE_WINDOWS_H # include # define sleep(x) Sleep(1000*(x)) #endif #if HAVE_ASSERT_H # include #endif #include "armci.h" #include "message.h" #define DIM1 5 #define DIM2 3 #ifdef __sun /* Solaris has shared memory shortages in the default system configuration */ # define DIM3 6 # define DIM4 5 # define DIM5 4 #elif defined(__alpha__) # define DIM3 8 # define DIM4 5 # define DIM5 6 #else # define DIM3 8 # define DIM4 9 # define DIM5 7 #endif #define DIM6 3 #define DIM7 2 #define OFF 1 #define EDIM1 (DIM1+OFF) #define EDIM2 (DIM2+OFF) #define EDIM3 (DIM3+OFF) #define EDIM4 (DIM4+OFF) #define EDIM5 (DIM5+OFF) #define EDIM6 (DIM6+OFF) #define EDIM7 (DIM7+OFF) #define DIMS 4 #define MAXDIMS 7 #define MAX_DIM_VAL 50 #define LOOP 200 #define BASE 100. #define MAXPROC 128 #define TIMES 100 # define ELEMS 200 /***************************** macros ************************/ #define COPY(src, dst, bytes) memcpy((dst),(src),(bytes)) #define ARMCI_MAX(a,b) (((a) >= (b)) ? (a) : (b)) #define ARMCI_MIN(a,b) (((a) <= (b)) ? (a) : (b)) #define ARMCI_ABS(a) (((a) <0) ? -(a) : (a)) #define ROW 65536 #define COL ROW /* square matrices only for the time being */ /***************************** global data *******************/ int me, nproc; short int fortran_indexing=0; static int proc_row_list[MAXPROC];/*no of rows owned by each process - accumulated*/ static int proc_nz_list[MAXPROC]; /*no of non-zeros owned by each process */ void create_array(void *a[], int elem_size, int ndim, int dims[]) { int bytes=elem_size, i, rc; assert(ndim<=MAXDIMS); for(i=0;i= local_nz_acc) { proc_row_list[proc_id++] = i+1; local_nz_acc = local_nz*(proc_id+1); if(proc_id == nproc-1) local_nz_acc = non_zero; if(me==0 && proc_id ROW) ARMCI_Error("order is greater than defined variable ROW", ROW); } len = sizeof(int); armci_msg_brdcst(n, len, 0); /* Broad cast number of non_zeros */ if(me==0) fscanf(fp, "%d", non_zero); armci_msg_brdcst(non_zero, len, 0); /* Broadcast row indices */ len = (*n+1)*sizeof(int); row_ind_tmp = (int *)malloc(len); if(me==0)for(i=0; i<*n+1; i++) { fscanf(fp, "%d", &row_ind_tmp[i]); if(fortran_indexing) --row_ind_tmp[i]; } armci_msg_brdcst(row_ind_tmp, len, 0); load_balance(*n, *non_zero, row_ind_tmp); /* find how much temporary storage is needed at the maximum */ if(me==0) { for(max=-1,j=0;j j) return (1); if (i < j) return (-1); return (0); } static int count = -1; static armci_hdl_t gHandle[MAXPROC]; static int prev_proc = -1; static void get_data(int n, int start, int end, double *vec_local, double **vec) { int i, j, rc, bytes, offset; int proc_start, proc_end, idx_start, idx_end; proc_start = proc_end = -1; for(i=0; istart) proc_start = i; if(proc_end<0 && proc_row_list[i]>end) proc_end = i; } if(proc_start<0 || proc_end<0) ARMCI_Error("Invalid Process Ids", -1); for(i=proc_start; i<=proc_end; i++) { if(i==proc_start) idx_start = start; else { if(i==0) idx_start=0; else idx_start = proc_row_list[i-1];} if(i==proc_end) idx_end = end; else idx_end = proc_row_list[i]-1; if(i!=prev_proc) { ++count; prev_proc = i; ARMCI_INIT_HANDLE(&gHandle[count]); ARMCI_SET_AGGREGATE_HANDLE(&gHandle[count]); } if(i==0) offset=0; else offset = proc_row_list[i-1]; if(i==me) { /* local */ for(j=idx_start; j<=idx_end; j++) vec_local[j] = vec[me][j-offset]; } else { /* remote */ bytes = (idx_end-idx_start+1)*sizeof(double); vec_local[idx_start] = -1; #if 0 if((rc=ARMCI_Get(&vec[i][idx_start-offset], &vec_local[idx_start], bytes, i))) #else if((rc=ARMCI_NbGet(&vec[i][idx_start-offset], &vec_local[idx_start], bytes, i, &gHandle[count]))) #endif ARMCI_Error("armci_nbget failed\n",rc); } } } static void sparse_multiply(int n, int non_zero, int *row_ind, int **col_ind, double **values, double **vec, double **svec) { int i, j, k, num_elements, offset, *tmp_indices; double start_time, comm_time, comp_time, v, vec_local[COL]; int start, end, prev, nrows, idx; #if 0 /* ---- Sequential Case ---- */ for(i=0; iprev+1) { end = prev; get_data(n, start, end, vec_local, vec); start = prev = tmp_indices[i]; } else prev = tmp_indices[i]; } get_data(n, start, prev, vec_local, vec); #if 1 if(count>=0) for(i=0; i<=count; i++) ARMCI_Wait(&gHandle[i]); #endif comm_time = armci_timer() - start_time; start_time = armci_timer(); /* Perform Matrix-Vector multiply and store the result in solution vector - "svec[]" */ if(me==0) { nrows = proc_row_list[me]; offset = row_ind[0]; } else { nrows = proc_row_list[me]-proc_row_list[me-1]; offset = row_ind[proc_row_list[me-1]]; } /* printf("%d: My total Work = %d\n", me, nrows); */ for(i=0; iMAXPROC && me==0) ARMCI_Error("Test works for up to %d processors\n",MAXPROC); if(me==0){ printf("ARMCI test program (%d processes)\n",nproc); fflush(stdout); sleep(1); } ARMCI_Init(); if(me==0){ printf("\n Performing Sparse Matrix-Vector Multiplication ...\n\n"); fflush(stdout); } test_sparse(); ARMCI_AllFence(); armci_msg_barrier(); if(me==0){printf("\nSuccess!!\n"); fflush(stdout);} sleep(2); armci_msg_barrier(); ARMCI_Finalize(); armci_msg_finalize(); return(0); }