ozblas-1.2a17/0000755000175200017520000000000013736552046013464 5ustar mukunokimukunokiozblas-1.2a17/include/0000755000175200017520000000000013725177602015106 5ustar mukunokimukunokiozblas-1.2a17/include/cuozblas.h0000644000175200017520000001522513725177602017106 0ustar mukunokimukunoki#ifndef CUOZBLAS_H #define CUOZBLAS_H #include #include typedef struct { cublasHandle_t ch; cusparseHandle_t csh; char* devWorkCommon; // for except split data, 128bytes char* devWork; // for split data char* hstBatchAddr; char* devBatchAddr; size_t workSizeBytes; size_t memAddr; uint32_t numSplitArraysMax; uint32_t fastModeFlag; uint32_t reproModeFlag; uint32_t sumModeFlag; // exec info uint32_t numSplitArraysA; uint32_t numSplitArraysB; uint32_t numSplitArraysC; uint32_t lastFlagA; uint32_t lastFlagB; // for SpMV in iterative solvers size_t memMaskSplitA; uint32_t numSplitArraysA_; int32_t splitShift; // tuning parameter (optional) float sparseThreshold; uint32_t useGemmFlag; uint32_t useMyGemmFlag; uint32_t useBatchedGemmFlag; uint32_t trueres; uint32_t verbose; double t_SplitA; double t_SplitB; double t_comp; double t_sum; double t_total; // for CG breakdown double t_SplitMat_total; double t_SplitVec_total; double t_Sum_total; double t_AXPY_SCAL_total; double t_DOT_NRM2_total; double t_SpMV_SpMM_total; } cuozblasHandle_t; // helper routines extern void cuozblasProfilingSetting (cuozblasHandle_t*, uint32_t, uint32_t, uint32_t); extern void cuozblasProfilingSetting2 (cuozblasHandle_t*, float); extern void cuozblasCreate (cuozblasHandle_t*, size_t, uint32_t, uint32_t, uint32_t, uint32_t); extern void cuozblasSetNumSplitArraysMax (cuozblasHandle_t*, uint32_t); extern void cuozblasSetFastMode (cuozblasHandle_t*, uint32_t); extern void cuozblasSetReproMode (cuozblasHandle_t*, uint32_t); extern void cuozblasSetSumMode (cuozblasHandle_t*, uint32_t); extern uint32_t cuozblasGetNumSplitArraysMax (cuozblasHandle_t*); extern uint32_t cuozblasGetFastMode (cuozblasHandle_t*); extern uint32_t cuozblasGetReproMode (cuozblasHandle_t*); extern uint32_t cuozblasGetSumMode (cuozblasHandle_t*); extern void cuozblasDestroy (cuozblasHandle_t*); // BLAS routines (double) extern int32_t cuozblasDgemm (cuozblasHandle_t*, const cublasOperation_t, const cublasOperation_t, const int32_t, const int32_t, const int32_t, const double*, const double*, const int32_t, const double*, const int32_t, const double*, double*, const int32_t); extern int32_t cuozblasDgemv (cuozblasHandle_t*, const cublasOperation_t, const int32_t, const int32_t, const double*, const double*, const int32_t, const double*, const int32_t, const double*, double*, const int32_t); extern int32_t cuozblasDdot (cuozblasHandle_t*, const int32_t, const double*, const int32_t, const double*, const int32_t, double*); extern int32_t cuozblasDnrm2 (cuozblasHandle_t*, const int32_t, const double*, const int32_t, double*); extern int32_t cuozblasDaxpy (cuozblasHandle_t*, const int32_t, const double*, const double*, const int32_t, double*, const int32_t); extern int32_t cuozblasDcsrmv (cuozblasHandle_t*, const cusparseOperation_t, const int32_t, const int32_t, const int32_t, const double*, const cusparseMatDescr_t, const double*, const int32_t*, const int32_t*, const double*, const double*, double*); extern int32_t cuozblasDcsrmvSplitA (cuozblasHandle_t*, const cusparseOperation_t, const int32_t, const int32_t, const int32_t, const cusparseMatDescr_t, const double*, const int32_t*, const int32_t*, double**); extern int32_t cuozblasDcg (cuozblasHandle_t *oh, const cusparseOperation_t tranA, const int32_t dimN, const int32_t dimNNZ, const cusparseMatDescr_t descrA, const double *matA, const int32_t *matArowptr, const int32_t *matAcolind, const double *vecB, double *vecX, int32_t maxiter, double tol); // BLAS routines (float) extern int32_t cuozblasSgemm (cuozblasHandle_t*, const cublasOperation_t, const cublasOperation_t, const int32_t, const int32_t, const int32_t, const float*, const float*, const int32_t, const float*, const int32_t, const float*, float*, const int32_t); extern int32_t cuozblasSgemv (cuozblasHandle_t*, const cublasOperation_t, const int32_t, const int32_t, const float*, const float*, const int32_t, const float*, const int32_t, const float*, float*, const int32_t); extern int32_t cuozblasSdot (cuozblasHandle_t*, const int32_t, const float*, const int32_t, const float*, const int32_t, float*); extern int32_t cuozblasSnrm2 (cuozblasHandle_t*, const int32_t, const float*, const int32_t, float*); extern int32_t cuozblasSaxpy (cuozblasHandle_t*, const int32_t, const float*, const float*, const int32_t, float*, const int32_t); extern int32_t cuozblasScsrmv (cuozblasHandle_t*, const cusparseOperation_t, const int32_t, const int32_t, const int32_t, const float*, const cusparseMatDescr_t, const float*, const int32_t*, const int32_t*, const float*, const float*, float*); extern int32_t cuozblasScsrmvSplitA (cuozblasHandle_t*, const cusparseOperation_t, const int32_t, const int32_t, const int32_t, const cusparseMatDescr_t, const float*, const int32_t*, const int32_t*, float**); extern int32_t cuozblasScg (cuozblasHandle_t *oh, const cusparseOperation_t tranA, const int32_t dimN, const int32_t dimNNZ, const cusparseMatDescr_t descrA, const float *matA, const int32_t *matArowptr, const int32_t *matAcolind, const float *vecB, float *vecX, int32_t maxiter, double tol); // BLAS routines (float-double) extern int32_t cuozblasSDgemm (cuozblasHandle_t*, const cublasOperation_t, const cublasOperation_t, const int32_t, const int32_t, const int32_t, const float*, const float*, const int32_t, const float*, const int32_t, const float*, float*, const int32_t); extern int32_t cuozblasSDgemv (cuozblasHandle_t*, const cublasOperation_t, const int32_t, const int32_t, const float*, const float*, const int32_t, const float*, const int32_t, const float*, float*, const int32_t); extern int32_t cuozblasSDdot (cuozblasHandle_t*, const int32_t, const float*, const int32_t, const float*, const int32_t, float*); extern int32_t cuozblasSDnrm2 (cuozblasHandle_t*, const int32_t, const float*, const int32_t, float*); extern int32_t cuozblasSDaxpy (cuozblasHandle_t*, const int32_t, const float*, const float*, const int32_t, float*, const int32_t); extern int32_t cuozblasSDcsrmv (cuozblasHandle_t*, const cusparseOperation_t, const int32_t, const int32_t, const int32_t, const float*, const cusparseMatDescr_t, const float*, const int32_t*, const int32_t*, const float*, const float*, float*); extern int32_t cuozblasSDcsrmvSplitA (cuozblasHandle_t*, const cusparseOperation_t, const int32_t, const int32_t, const int32_t, const cusparseMatDescr_t, const float*, const int32_t*, const int32_t*, double**); extern int32_t cuozblasSDcg (cuozblasHandle_t *oh, const cusparseOperation_t tranA, const int32_t dimN, const int32_t dimNNZ, const cusparseMatDescr_t descrA, const float *matA, const int32_t *matArowptr, const int32_t *matAcolind, const float *vecB, float *vecX, int32_t maxiter, double tol); #endif ozblas-1.2a17/include/ozblas.h0000644000175200017520000001227713725177602016562 0ustar mukunokimukunoki#ifndef OZBLAS_H #define OZBLAS_H typedef struct { char* devWork; char* devBatchAddr; uint64_t workSizeBytes; uint numSplitArraysMax; uint fastModeFlag; uint reproModeFlag; uint sumModeFlag; uint64_t memAddr; // exec info uint numSplitArraysA; uint numSplitArraysB; uint numSplitArraysC; uint lastFlagA; uint lastFlagB; // for SpMV in iterative solvers size_t memMaskSplitA; uint32_t numSplitArraysA_; int32_t splitShift; uint trueres; uint verbose; uint useGemmFlag; uint useMyGemmFlag; uint useBatchedGemmFlag; double t_SplitA; double t_SplitB; double t_comp; double t_sum; double t_total; // for CG breakdown double t_SplitMat_total; double t_SplitVec_total; double t_Sum_total; double t_AXPY_SCAL_total; double t_DOT_NRM2_total; double t_SpMV_SpMM_total; } ozblasHandle_t; // helper routines extern void ozblasProfilingSetting (ozblasHandle_t*, uint, uint, uint); extern void ozblasCreate (ozblasHandle_t*, uint64_t, uint, uint, uint, uint); extern void ozblasSetNumSplitArraysMax (ozblasHandle_t*, uint); extern void ozblasSetFastMode (ozblasHandle_t*, uint); extern void ozblasSetReproMode (ozblasHandle_t*, uint); extern void ozblasSetSumMode (ozblasHandle_t*, uint); extern uint ozblasGetNumSplitArraysMax (ozblasHandle_t*); extern uint ozblasGetFastMode (ozblasHandle_t*); extern uint ozblasGetReproMode (ozblasHandle_t*); extern uint ozblasGetSumMode (ozblasHandle_t*); extern void ozblasDestroy (ozblasHandle_t*); // BLAS routines (double) extern int ozblasDgemm (ozblasHandle_t*, const char, const char, const int, const int, const int, const double, const double*, const int, const double*, const int, const double, double*, const int); extern int ozblasDgemv (ozblasHandle_t*, const char, const int, const int, const double, const double*, const int, const double*, const int, const double, double*, const int); extern double ozblasDdot (ozblasHandle_t*, const int, const double*, const int, const double*, const int); extern double ozblasDnrm2 (ozblasHandle_t*, const int, const double*, const int); extern int ozblasDaxpy (ozblasHandle_t*, const int, const double, const double*, const int, double*, const int); extern int ozblasDcsrmv (ozblasHandle_t*, const char, const int, const int, const int, const double, const char*, const double*, const int*, const int*, const double*, const double, double*); extern int ozblasDcsrmvSplitA (ozblasHandle_t*, const char, const int, const int, const int, const char *, const double*, const int*, const int*, double**); extern int ozblasDcg (ozblasHandle_t *oh, const char tranA, const int dimN, const int dimNNZ, const char* descrA, const double *matA, const int *matArowptr, const int *matAcolind, const double *vecB, double *vecX, int maxiter, double tol); // BLAS routines (float) extern int ozblasSgemm (ozblasHandle_t*, const char, const char, const int, const int, const int, const float, const float*, const int, const float*, const int, const float, float*, const int); extern int ozblasSgemv (ozblasHandle_t*, const char, const int, const int, const float, const float*, const int, const float*, const int, const float, float*, const int); extern float ozblasSdot (ozblasHandle_t*, const int, const float*, const int, const float*, const int); extern float ozblasSnrm2 (ozblasHandle_t*, const int, const float*, const int); extern int ozblasSaxpy (ozblasHandle_t*, const int, const float, const float*, const int, float*, const int); extern int ozblasScsrmv (ozblasHandle_t*, const char, const int, const int, const int, const float, const char*, const float*, const int*, const int*, const float*, const float, float*); extern int ozblasScsrmvSplitA (ozblasHandle_t*, const char, const int, const int, const int, const char *, const float*, const int*, const int*, float**); extern int ozblasScg (ozblasHandle_t *oh, const char tranA, const int dimN, const int dimNNZ, const char* descrA, const float *matA, const int *matArowptr, const int *matAcolind, const float *vecB, float *vecX, int maxiter, double tol); // BLAS routines (float-double) extern int ozblasSDgemm (ozblasHandle_t*, const char, const char, const int, const int, const int, const float, const float*, const int, const float*, const int, const float, float*, const int); extern int ozblasSDgemv (ozblasHandle_t*, const char, const int, const int, const float, const float*, const int, const float*, const int, const float, float*, const int); extern float ozblasSDdot (ozblasHandle_t*, const int, const float*, const int, const float*, const int); extern float ozblasSDnrm2 (ozblasHandle_t*, const int, const float*, const int); extern int ozblasSDaxpy (ozblasHandle_t*, const int, const float, const float*, const int, float*, const int); extern int ozblasSDcsrmv (ozblasHandle_t*, const char, const int, const int, const int, const float, const char*, const float*, const int*, const int*, const float*, const float, float*); extern int ozblasSDcsrmvSplitA (ozblasHandle_t*, const char, const int, const int, const int, const char *, const float*, const int*, const int*, float**); extern int ozblasSDcg (ozblasHandle_t *oh, const char tranA, const int dimN, const int dimNNZ, const char* descrA, const float *matA, const int *matArowptr, const int *matAcolind, const float *vecB, float *vecX, int maxiter, double tol); #endif ozblas-1.2a17/lib/0000755000175200017520000000000013736552053014230 5ustar mukunokimukunokiozblas-1.2a17/Makefile0000644000175200017520000000053413725177602015125 0ustar mukunokimukunokiMAKE = make #--no-print-directory include make.inc ifeq ($(CUDA),yes) MAKE_DIRS += src/cuda endif ifeq ($(CPU),yes) MAKE_DIRS += src/cpu endif MAKE_DIRS += testing default: @for subdir in $(MAKE_DIRS) ; do \ (cd $$subdir && $(MAKE)) ;\ done clean: @for subdir in $(MAKE_DIRS) ; do \ (cd $$subdir && $(MAKE) clean) ;\ done ozblas-1.2a17/src/0000755000175200017520000000000013725177602014252 5ustar mukunokimukunokiozblas-1.2a17/src/cuda/0000755000175200017520000000000013736552053015165 5ustar mukunokimukunokiozblas-1.2a17/src/cuda/blas/0000755000175200017520000000000013725177602016107 5ustar mukunokimukunokiozblas-1.2a17/src/cuda/blas/mublas_dot_3_kernel.cu0000644000175200017520000000457113725177602022362 0ustar mukunokimukunoki__global__ void mublas_sum_3_device_s1 ( const size_t n, FP_CTYPE *w, const size_t ldw ) { const size_t iTx = threadIdx.x; const size_t nTx = DOTGEMM_NTX;//blockDim.x; size_t i, ix, iy; extern __shared__ FP_CTYPE Ts1[]; FP_CTYPE Tr1[3*3] = {0.}; for (i = iTx; i < n; i += nTx) { #pragma unroll for (ix = 0; ix < 3; ix++) { #pragma unroll for (iy = 0; iy < 3; iy++) Tr1[iy*3+ix] += w[ldw*(iy*3+ix)+i]; } } #pragma unroll for (ix = 0; ix < 3; ix++) { #pragma unroll for (iy = 0; iy < 3; iy++) Ts1[nTx*(iy*3+ix)+iTx] = Tr1[iy*3+ix]; } __syncthreads (); #pragma unroll for (i = nTx/2; i > 0; i >>= 1) { if (iTx < i) { #pragma unroll for (ix = 0; ix < 3; ix++) { #pragma unroll for (iy = 0; iy < 3; iy++) Ts1[nTx*(iy*3+ix)+iTx] += Ts1[nTx*(iy*3+ix)+iTx+i]; } } __syncthreads (); } if (iTx == 0) { #pragma unroll for (ix = 0; ix < 3; ix++) { #pragma unroll for (iy = 0; iy < 3; iy++) w[iy*3+ix] = Ts1[nTx*(iy*3+ix)]; } } } __global__ void mublas_dot_3_device_s1 ( const size_t n, const FP_CTYPE * __restrict__ x, const size_t ldx, const FP_CTYPE * __restrict__ y, const size_t ldy, FP_CTYPE *w, const size_t ldw ) { const size_t iTx = threadIdx.x; const size_t iBx = blockIdx.x; const size_t nBx = gridDim.x; const size_t nTx = DOTGEMM_NTX;//blockDim.x; const size_t addrx = iBx * nTx + iTx; size_t i, ix, iy; extern __shared__ FP_CTYPE Ts1[]; FP_CTYPE Xr1[3], Yr1[3]; FP_CTYPE Tr1[3*3] = {0.}; for (i = addrx; i < n; i += nTx * nBx) { #pragma unroll for (ix = 0; ix < 3; ix++) Xr1[ix] = x[ix * ldx + i]; #pragma unroll for (iy = 0; iy < 3; iy++) Yr1[iy] = y[iy * ldy + i]; #pragma unroll for (ix = 0; ix < 3; ix++) { #pragma unroll for (iy = 0; iy < 3; iy++) Tr1[iy*3+ix] += Xr1[ix] * Yr1[iy]; } } #pragma unroll for (ix = 0; ix < 3; ix++) { #pragma unroll for (iy = 0; iy < 3; iy++) Ts1[nTx*(iy*3+ix)+iTx] = Tr1[iy*3+ix]; } __syncthreads (); #pragma unroll for (i = nTx/2; i > 0; i >>= 1) { if (iTx < i) { #pragma unroll for (ix = 0; ix < 3; ix++) { #pragma unroll for (iy = 0; iy < 3; iy++) Ts1[nTx*(iy*3+ix)+iTx] += Ts1[nTx*(iy*3+ix)+iTx+i]; } } __syncthreads (); } if (iTx == 0) { #pragma unroll for (ix = 0; ix < 3; ix++) { #pragma unroll for (iy = 0; iy < 3; iy++) w[ldw*(iy*3+ix)+iBx] = Ts1[nTx*(iy*3+ix)]; } } } ozblas-1.2a17/src/cuda/blas/mublas_dot_2_kernel.cu0000644000175200017520000000457113725177602022361 0ustar mukunokimukunoki__global__ void mublas_sum_2_device_s1 ( const size_t n, FP_CTYPE *w, const size_t ldw ) { const size_t iTx = threadIdx.x; const size_t nTx = DOTGEMM_NTX;//blockDim.x; size_t i, ix, iy; extern __shared__ FP_CTYPE Ts1[]; FP_CTYPE Tr1[2*2] = {0.}; for (i = iTx; i < n; i += nTx) { #pragma unroll for (ix = 0; ix < 2; ix++) { #pragma unroll for (iy = 0; iy < 2; iy++) Tr1[iy*2+ix] += w[ldw*(iy*2+ix)+i]; } } #pragma unroll for (ix = 0; ix < 2; ix++) { #pragma unroll for (iy = 0; iy < 2; iy++) Ts1[nTx*(iy*2+ix)+iTx] = Tr1[iy*2+ix]; } __syncthreads (); #pragma unroll for (i = nTx/2; i > 0; i >>= 1) { if (iTx < i) { #pragma unroll for (ix = 0; ix < 2; ix++) { #pragma unroll for (iy = 0; iy < 2; iy++) Ts1[nTx*(iy*2+ix)+iTx] += Ts1[nTx*(iy*2+ix)+iTx+i]; } } __syncthreads (); } if (iTx == 0) { #pragma unroll for (ix = 0; ix < 2; ix++) { #pragma unroll for (iy = 0; iy < 2; iy++) w[iy*2+ix] = Ts1[nTx*(iy*2+ix)]; } } } __global__ void mublas_dot_2_device_s1 ( const size_t n, const FP_CTYPE * __restrict__ x, const size_t ldx, const FP_CTYPE * __restrict__ y, const size_t ldy, FP_CTYPE *w, const size_t ldw ) { const size_t iTx = threadIdx.x; const size_t iBx = blockIdx.x; const size_t nBx = gridDim.x; const size_t nTx = DOTGEMM_NTX;//blockDim.x; const size_t addrx = iBx * nTx + iTx; size_t i, ix, iy; extern __shared__ FP_CTYPE Ts1[]; FP_CTYPE Xr1[2], Yr1[2]; FP_CTYPE Tr1[2*2] = {0.}; for (i = addrx; i < n; i += nTx * nBx) { #pragma unroll for (ix = 0; ix < 2; ix++) Xr1[ix] = x[ix * ldx + i]; #pragma unroll for (iy = 0; iy < 2; iy++) Yr1[iy] = y[iy * ldy + i]; #pragma unroll for (ix = 0; ix < 2; ix++) { #pragma unroll for (iy = 0; iy < 2; iy++) Tr1[iy*2+ix] += Xr1[ix] * Yr1[iy]; } } #pragma unroll for (ix = 0; ix < 2; ix++) { #pragma unroll for (iy = 0; iy < 2; iy++) Ts1[nTx*(iy*2+ix)+iTx] = Tr1[iy*2+ix]; } __syncthreads (); #pragma unroll for (i = nTx/2; i > 0; i >>= 1) { if (iTx < i) { #pragma unroll for (ix = 0; ix < 2; ix++) { #pragma unroll for (iy = 0; iy < 2; iy++) Ts1[nTx*(iy*2+ix)+iTx] += Ts1[nTx*(iy*2+ix)+iTx+i]; } } __syncthreads (); } if (iTx == 0) { #pragma unroll for (ix = 0; ix < 2; ix++) { #pragma unroll for (iy = 0; iy < 2; iy++) w[ldw*(iy*2+ix)+iBx] = Ts1[nTx*(iy*2+ix)]; } } } ozblas-1.2a17/src/cuda/blas/mublas_dot.cu0000644000175200017520000000576413725177602020605 0ustar mukunokimukunoki#define DOTGEMM_NTX 256 #define DOTGEMM_NTX_SMALL 128 #define DOTGEMM_NTX_XSMALL 64 #define DOTGEMM_NTX_XXSMALL 32 #include "mublas_dot_2_kernel.cu" #include "mublas_dot_3_kernel.cu" #include "mublas_dot_4_kernel.cu" #include "mublas_dot_5_kernel.cu" #include "mublas_dot_6_kernel.cu" #include "mublas_dot_7_kernel.cu" #include "mublas_dot_8_kernel.cu" #include "mublas_dot_9_kernel.cu" #include "mublas_dot_10_kernel.cu" __host__ __inline__ size_t ceilingEx (const float n, const size_t c) { return (size_t)ceil(n/c)*c; } __host__ int32_t MuMDOT ( cublasHandle_t ch, const size_t n, const size_t sx, const size_t sy, const FP_CTYPE *x, const size_t ldx, const FP_CTYPE *y, const size_t ldy, FP_CTYPE *result, const size_t ldc, FP_CTYPE *w ) { void (*kernelMain) (const size_t, const FP_CTYPE*, const size_t, const FP_CTYPE*, const size_t, FP_CTYPE*, const size_t); void (*kernelSub) (const size_t, FP_CTYPE*, const size_t); if (sx == sy && sx == 1) { cublasSetPointerMode(ch, CUBLAS_POINTER_MODE_DEVICE); CuDOT (ch, n, x, 1, y, 1, result); cublasSetPointerMode(ch, CUBLAS_POINTER_MODE_HOST); } else if (sx == sy && max(sx,sy) <= 10) { switch (sx) { case 2: kernelMain = &mublas_dot_2_device_s1; kernelSub = &mublas_sum_2_device_s1; break; case 3: kernelMain = &mublas_dot_3_device_s1; kernelSub = &mublas_sum_3_device_s1; break; case 4: kernelMain = &mublas_dot_4_device_s1; kernelSub = &mublas_sum_4_device_s1; break; case 5: kernelMain = &mublas_dot_5_device_s1; kernelSub = &mublas_sum_5_device_s1; break; case 6: kernelMain = &mublas_dot_6_device_s1; kernelSub = &mublas_sum_6_device_s1; break; case 7: kernelMain = &mublas_dot_7_device_s1; kernelSub = &mublas_sum_7_device_s1; break; case 8: kernelMain = &mublas_dot_8_device_s1; kernelSub = &mublas_sum_8_device_s1; break; case 9: kernelMain = &mublas_dot_9_device_s1; kernelSub = &mublas_sum_9_device_s1; break; case 10: kernelMain = &mublas_dot_10_device_s1; kernelSub = &mublas_sum_10_device_s1; break; } size_t ntx, nbx; dim3 threads, grid; ntx = DOTGEMM_NTX; if (sx > 4) ntx = DOTGEMM_NTX_SMALL; if (sx > 6) ntx = DOTGEMM_NTX_XSMALL; if (sx > 9) ntx = DOTGEMM_NTX_XXSMALL; nbx = min ((int)ceil((float)n/ntx), SPLIT_VEC_NBX); threads = dim3 (ntx); grid = dim3 (nbx); size_t ShmPerBlk = ceilingEx ((float)(sx * sy * ntx * sizeof(FP_CTYPE)), 256); //256:Shm Unit Size size_t ldw = nbx; kernelMain <<< grid, threads, ShmPerBlk >>> (n, x, ldx, y, ldy, w, ldw); grid = dim3 (1); kernelSub <<< grid, threads, ShmPerBlk >>> (nbx, w, ldw); cudaMemcpy (result, w, sizeof(FP_CTYPE)*ldc*sy, cudaMemcpyDeviceToDevice); } else { FP_CTYPE alpha_ = 1., beta_ = 0.; cublasOperation_t tranA = CUBLAS_OP_N; cublasOperation_t tranAt = CUBLAS_OP_T; cublasSetPointerMode(ch, CUBLAS_POINTER_MODE_HOST); CuGEMM (ch, tranAt, tranA, sx, sy, n, &alpha_, x, ldx, y, ldy, &beta_, result, sx); } return 0; } ozblas-1.2a17/src/cuda/blas/mublas_dot_10_kernel.cu0000644000175200017520000000466213725177602022441 0ustar mukunokimukunoki__global__ void mublas_sum_10_device_s1 ( const size_t n, FP_CTYPE *w, const size_t ldw ) { const size_t iTx = threadIdx.x; const size_t nTx = DOTGEMM_NTX_XXSMALL;//blockDim.x; size_t i, ix, iy; extern __shared__ FP_CTYPE Ts1[]; FP_CTYPE Tr1[10*10] = {0.}; for (i = iTx; i < n; i += nTx) { #pragma unroll for (ix = 0; ix < 10; ix++) { #pragma unroll for (iy = 0; iy < 10; iy++) Tr1[iy*10+ix] += w[ldw*(iy*10+ix)+i]; } } #pragma unroll for (ix = 0; ix < 10; ix++) { #pragma unroll for (iy = 0; iy < 10; iy++) Ts1[nTx*(iy*10+ix)+iTx] = Tr1[iy*10+ix]; } __syncthreads (); #pragma unroll for (i = nTx/2; i > 0; i >>= 1) { if (iTx < i) { #pragma unroll for (ix = 0; ix < 10; ix++) { #pragma unroll for (iy = 0; iy < 10; iy++) Ts1[nTx*(iy*10+ix)+iTx] += Ts1[nTx*(iy*10+ix)+iTx+i]; } } __syncthreads (); } if (iTx == 0) { #pragma unroll for (ix = 0; ix < 10; ix++) { #pragma unroll for (iy = 0; iy < 10; iy++) w[iy*10+ix] = Ts1[nTx*(iy*10+ix)]; } } } __global__ void mublas_dot_10_device_s1 ( const size_t n, const FP_CTYPE * __restrict__ x, const size_t ldx, const FP_CTYPE * __restrict__ y, const size_t ldy, FP_CTYPE *w, const size_t ldw ) { const size_t iTx = threadIdx.x; const size_t iBx = blockIdx.x; const size_t nBx = gridDim.x; const size_t nTx = DOTGEMM_NTX_XXSMALL;//blockDim.x; const size_t addrx = iBx * nTx + iTx; size_t i, ix, iy; extern __shared__ FP_CTYPE Ts1[]; FP_CTYPE Xr1[10], Yr1[10]; FP_CTYPE Tr1[10*10] = {0.}; for (i = addrx; i < n; i += nTx * nBx) { #pragma unroll for (ix = 0; ix < 10; ix++) Xr1[ix] = x[ix * ldx + i]; #pragma unroll for (iy = 0; iy < 10; iy++) Yr1[iy] = y[iy * ldy + i]; #pragma unroll for (ix = 0; ix < 10; ix++) { #pragma unroll for (iy = 0; iy < 10; iy++) Tr1[iy*10+ix] += Xr1[ix] * Yr1[iy]; } } #pragma unroll for (ix = 0; ix < 10; ix++) { #pragma unroll for (iy = 0; iy < 10; iy++) Ts1[nTx*(iy*10+ix)+iTx] = Tr1[iy*10+ix]; } __syncthreads (); #pragma unroll for (i = nTx/2; i > 0; i >>= 1) { if (iTx < i) { #pragma unroll for (ix = 0; ix < 10; ix++) { #pragma unroll for (iy = 0; iy < 10; iy++) Ts1[nTx*(iy*10+ix)+iTx] += Ts1[nTx*(iy*10+ix)+iTx+i]; } } __syncthreads (); } if (iTx == 0) { #pragma unroll for (ix = 0; ix < 10; ix++) { #pragma unroll for (iy = 0; iy < 10; iy++) w[ldw*(iy*10+ix)+iBx] = Ts1[nTx*(iy*10+ix)]; } } } ozblas-1.2a17/src/cuda/blas/mublas_dot_9_kernel.cu0000644000175200017520000000460713725177602022370 0ustar mukunokimukunoki__global__ void mublas_sum_9_device_s1 ( const size_t n, FP_CTYPE *w, const size_t ldw ) { const size_t iTx = threadIdx.x; const size_t nTx = DOTGEMM_NTX_XSMALL;//blockDim.x; size_t i, ix, iy; extern __shared__ FP_CTYPE Ts1[]; FP_CTYPE Tr1[9*9] = {0.}; for (i = iTx; i < n; i += nTx) { #pragma unroll for (ix = 0; ix < 9; ix++) { #pragma unroll for (iy = 0; iy < 9; iy++) Tr1[iy*9+ix] += w[ldw*(iy*9+ix)+i]; } } #pragma unroll for (ix = 0; ix < 9; ix++) { #pragma unroll for (iy = 0; iy < 9; iy++) Ts1[nTx*(iy*9+ix)+iTx] = Tr1[iy*9+ix]; } __syncthreads (); #pragma unroll for (i = nTx/2; i > 0; i >>= 1) { if (iTx < i) { #pragma unroll for (ix = 0; ix < 9; ix++) { #pragma unroll for (iy = 0; iy < 9; iy++) Ts1[nTx*(iy*9+ix)+iTx] += Ts1[nTx*(iy*9+ix)+iTx+i]; } } __syncthreads (); } if (iTx == 0) { #pragma unroll for (ix = 0; ix < 9; ix++) { #pragma unroll for (iy = 0; iy < 9; iy++) w[iy*9+ix] = Ts1[nTx*(iy*9+ix)]; } } } __global__ void mublas_dot_9_device_s1 ( const size_t n, const FP_CTYPE * __restrict__ x, const size_t ldx, const FP_CTYPE * __restrict__ y, const size_t ldy, FP_CTYPE *w, const size_t ldw ) { const size_t iTx = threadIdx.x; const size_t iBx = blockIdx.x; const size_t nBx = gridDim.x; const size_t nTx = DOTGEMM_NTX_XSMALL;//blockDim.x; const size_t addrx = iBx * nTx + iTx; size_t i, ix, iy; extern __shared__ FP_CTYPE Ts1[]; FP_CTYPE Xr1[9], Yr1[9]; FP_CTYPE Tr1[9*9] = {0.}; for (i = addrx; i < n; i += nTx * nBx) { #pragma unroll for (ix = 0; ix < 9; ix++) Xr1[ix] = x[ix * ldx + i]; #pragma unroll for (iy = 0; iy < 9; iy++) Yr1[iy] = y[iy * ldy + i]; #pragma unroll for (ix = 0; ix < 9; ix++) { #pragma unroll for (iy = 0; iy < 9; iy++) Tr1[iy*9+ix] += Xr1[ix] * Yr1[iy]; } } #pragma unroll for (ix = 0; ix < 9; ix++) { #pragma unroll for (iy = 0; iy < 9; iy++) Ts1[nTx*(iy*9+ix)+iTx] = Tr1[iy*9+ix]; } __syncthreads (); #pragma unroll for (i = nTx/2; i > 0; i >>= 1) { if (iTx < i) { #pragma unroll for (ix = 0; ix < 9; ix++) { #pragma unroll for (iy = 0; iy < 9; iy++) Ts1[nTx*(iy*9+ix)+iTx] += Ts1[nTx*(iy*9+ix)+iTx+i]; } } __syncthreads (); } if (iTx == 0) { #pragma unroll for (ix = 0; ix < 9; ix++) { #pragma unroll for (iy = 0; iy < 9; iy++) w[ldw*(iy*9+ix)+iBx] = Ts1[nTx*(iy*9+ix)]; } } } ozblas-1.2a17/src/cuda/blas/mublas_dot_8_kernel.cu0000644000175200017520000000460713725177602022367 0ustar mukunokimukunoki__global__ void mublas_sum_8_device_s1 ( const size_t n, FP_CTYPE *w, const size_t ldw ) { const size_t iTx = threadIdx.x; const size_t nTx = DOTGEMM_NTX_XSMALL;//blockDim.x; size_t i, ix, iy; extern __shared__ FP_CTYPE Ts1[]; FP_CTYPE Tr1[8*8] = {0.}; for (i = iTx; i < n; i += nTx) { #pragma unroll for (ix = 0; ix < 8; ix++) { #pragma unroll for (iy = 0; iy < 8; iy++) Tr1[iy*8+ix] += w[ldw*(iy*8+ix)+i]; } } #pragma unroll for (ix = 0; ix < 8; ix++) { #pragma unroll for (iy = 0; iy < 8; iy++) Ts1[nTx*(iy*8+ix)+iTx] = Tr1[iy*8+ix]; } __syncthreads (); #pragma unroll for (i = nTx/2; i > 0; i >>= 1) { if (iTx < i) { #pragma unroll for (ix = 0; ix < 8; ix++) { #pragma unroll for (iy = 0; iy < 8; iy++) Ts1[nTx*(iy*8+ix)+iTx] += Ts1[nTx*(iy*8+ix)+iTx+i]; } } __syncthreads (); } if (iTx == 0) { #pragma unroll for (ix = 0; ix < 8; ix++) { #pragma unroll for (iy = 0; iy < 8; iy++) w[iy*8+ix] = Ts1[nTx*(iy*8+ix)]; } } } __global__ void mublas_dot_8_device_s1 ( const size_t n, const FP_CTYPE * __restrict__ x, const size_t ldx, const FP_CTYPE * __restrict__ y, const size_t ldy, FP_CTYPE *w, const size_t ldw ) { const size_t iTx = threadIdx.x; const size_t iBx = blockIdx.x; const size_t nBx = gridDim.x; const size_t nTx = DOTGEMM_NTX_XSMALL;//blockDim.x; const size_t addrx = iBx * nTx + iTx; size_t i, ix, iy; extern __shared__ FP_CTYPE Ts1[]; FP_CTYPE Xr1[8], Yr1[8]; FP_CTYPE Tr1[8*8] = {0.}; for (i = addrx; i < n; i += nTx * nBx) { #pragma unroll for (ix = 0; ix < 8; ix++) Xr1[ix] = x[ix * ldx + i]; #pragma unroll for (iy = 0; iy < 8; iy++) Yr1[iy] = y[iy * ldy + i]; #pragma unroll for (ix = 0; ix < 8; ix++) { #pragma unroll for (iy = 0; iy < 8; iy++) Tr1[iy*8+ix] += Xr1[ix] * Yr1[iy]; } } #pragma unroll for (ix = 0; ix < 8; ix++) { #pragma unroll for (iy = 0; iy < 8; iy++) Ts1[nTx*(iy*8+ix)+iTx] = Tr1[iy*8+ix]; } __syncthreads (); #pragma unroll for (i = nTx/2; i > 0; i >>= 1) { if (iTx < i) { #pragma unroll for (ix = 0; ix < 8; ix++) { #pragma unroll for (iy = 0; iy < 8; iy++) Ts1[nTx*(iy*8+ix)+iTx] += Ts1[nTx*(iy*8+ix)+iTx+i]; } } __syncthreads (); } if (iTx == 0) { #pragma unroll for (ix = 0; ix < 8; ix++) { #pragma unroll for (iy = 0; iy < 8; iy++) w[ldw*(iy*8+ix)+iBx] = Ts1[nTx*(iy*8+ix)]; } } } ozblas-1.2a17/src/cuda/blas/mublas_dot_7_kernel.cu0000644000175200017520000000460713725177602022366 0ustar mukunokimukunoki__global__ void mublas_sum_7_device_s1 ( const size_t n, FP_CTYPE *w, const size_t ldw ) { const size_t iTx = threadIdx.x; const size_t nTx = DOTGEMM_NTX_XSMALL;//blockDim.x; size_t i, ix, iy; extern __shared__ FP_CTYPE Ts1[]; FP_CTYPE Tr1[7*7] = {0.}; for (i = iTx; i < n; i += nTx) { #pragma unroll for (ix = 0; ix < 7; ix++) { #pragma unroll for (iy = 0; iy < 7; iy++) Tr1[iy*7+ix] += w[ldw*(iy*7+ix)+i]; } } #pragma unroll for (ix = 0; ix < 7; ix++) { #pragma unroll for (iy = 0; iy < 7; iy++) Ts1[nTx*(iy*7+ix)+iTx] = Tr1[iy*7+ix]; } __syncthreads (); #pragma unroll for (i = nTx/2; i > 0; i >>= 1) { if (iTx < i) { #pragma unroll for (ix = 0; ix < 7; ix++) { #pragma unroll for (iy = 0; iy < 7; iy++) Ts1[nTx*(iy*7+ix)+iTx] += Ts1[nTx*(iy*7+ix)+iTx+i]; } } __syncthreads (); } if (iTx == 0) { #pragma unroll for (ix = 0; ix < 7; ix++) { #pragma unroll for (iy = 0; iy < 7; iy++) w[iy*7+ix] = Ts1[nTx*(iy*7+ix)]; } } } __global__ void mublas_dot_7_device_s1 ( const size_t n, const FP_CTYPE * __restrict__ x, const size_t ldx, const FP_CTYPE * __restrict__ y, const size_t ldy, FP_CTYPE *w, const size_t ldw ) { const size_t iTx = threadIdx.x; const size_t iBx = blockIdx.x; const size_t nBx = gridDim.x; const size_t nTx = DOTGEMM_NTX_XSMALL;//blockDim.x; const size_t addrx = iBx * nTx + iTx; size_t i, ix, iy; extern __shared__ FP_CTYPE Ts1[]; FP_CTYPE Xr1[7], Yr1[7]; FP_CTYPE Tr1[7*7] = {0.}; for (i = addrx; i < n; i += nTx * nBx) { #pragma unroll for (ix = 0; ix < 7; ix++) Xr1[ix] = x[ix * ldx + i]; #pragma unroll for (iy = 0; iy < 7; iy++) Yr1[iy] = y[iy * ldy + i]; #pragma unroll for (ix = 0; ix < 7; ix++) { #pragma unroll for (iy = 0; iy < 7; iy++) Tr1[iy*7+ix] += Xr1[ix] * Yr1[iy]; } } #pragma unroll for (ix = 0; ix < 7; ix++) { #pragma unroll for (iy = 0; iy < 7; iy++) Ts1[nTx*(iy*7+ix)+iTx] = Tr1[iy*7+ix]; } __syncthreads (); #pragma unroll for (i = nTx/2; i > 0; i >>= 1) { if (iTx < i) { #pragma unroll for (ix = 0; ix < 7; ix++) { #pragma unroll for (iy = 0; iy < 7; iy++) Ts1[nTx*(iy*7+ix)+iTx] += Ts1[nTx*(iy*7+ix)+iTx+i]; } } __syncthreads (); } if (iTx == 0) { #pragma unroll for (ix = 0; ix < 7; ix++) { #pragma unroll for (iy = 0; iy < 7; iy++) w[ldw*(iy*7+ix)+iBx] = Ts1[nTx*(iy*7+ix)]; } } } ozblas-1.2a17/src/cuda/blas/mublas_dot_6_kernel.cu0000644000175200017520000000460513725177602022363 0ustar mukunokimukunoki__global__ void mublas_sum_6_device_s1 ( const size_t n, FP_CTYPE *w, const size_t ldw ) { const size_t iTx = threadIdx.x; const size_t nTx = DOTGEMM_NTX_SMALL;//blockDim.x; size_t i, ix, iy; extern __shared__ FP_CTYPE Ts1[]; FP_CTYPE Tr1[6*6] = {0.}; for (i = iTx; i < n; i += nTx) { #pragma unroll for (ix = 0; ix < 6; ix++) { #pragma unroll for (iy = 0; iy < 6; iy++) Tr1[iy*6+ix] += w[ldw*(iy*6+ix)+i]; } } #pragma unroll for (ix = 0; ix < 6; ix++) { #pragma unroll for (iy = 0; iy < 6; iy++) Ts1[nTx*(iy*6+ix)+iTx] = Tr1[iy*6+ix]; } __syncthreads (); #pragma unroll for (i = nTx/2; i > 0; i >>= 1) { if (iTx < i) { #pragma unroll for (ix = 0; ix < 6; ix++) { #pragma unroll for (iy = 0; iy < 6; iy++) Ts1[nTx*(iy*6+ix)+iTx] += Ts1[nTx*(iy*6+ix)+iTx+i]; } } __syncthreads (); } if (iTx == 0) { #pragma unroll for (ix = 0; ix < 6; ix++) { #pragma unroll for (iy = 0; iy < 6; iy++) w[iy*6+ix] = Ts1[nTx*(iy*6+ix)]; } } } __global__ void mublas_dot_6_device_s1 ( const size_t n, const FP_CTYPE * __restrict__ x, const size_t ldx, const FP_CTYPE * __restrict__ y, const size_t ldy, FP_CTYPE *w, const size_t ldw ) { const size_t iTx = threadIdx.x; const size_t iBx = blockIdx.x; const size_t nBx = gridDim.x; const size_t nTx = DOTGEMM_NTX_SMALL;//blockDim.x; const size_t addrx = iBx * nTx + iTx; size_t i, ix, iy; extern __shared__ FP_CTYPE Ts1[]; FP_CTYPE Xr1[6], Yr1[6]; FP_CTYPE Tr1[6*6] = {0.}; for (i = addrx; i < n; i += nTx * nBx) { #pragma unroll for (ix = 0; ix < 6; ix++) Xr1[ix] = x[ix * ldx + i]; #pragma unroll for (iy = 0; iy < 6; iy++) Yr1[iy] = y[iy * ldy + i]; #pragma unroll for (ix = 0; ix < 6; ix++) { #pragma unroll for (iy = 0; iy < 6; iy++) Tr1[iy*6+ix] += Xr1[ix] * Yr1[iy]; } } #pragma unroll for (ix = 0; ix < 6; ix++) { #pragma unroll for (iy = 0; iy < 6; iy++) Ts1[nTx*(iy*6+ix)+iTx] = Tr1[iy*6+ix]; } __syncthreads (); #pragma unroll for (i = nTx/2; i > 0; i >>= 1) { if (iTx < i) { #pragma unroll for (ix = 0; ix < 6; ix++) { #pragma unroll for (iy = 0; iy < 6; iy++) Ts1[nTx*(iy*6+ix)+iTx] += Ts1[nTx*(iy*6+ix)+iTx+i]; } } __syncthreads (); } if (iTx == 0) { #pragma unroll for (ix = 0; ix < 6; ix++) { #pragma unroll for (iy = 0; iy < 6; iy++) w[ldw*(iy*6+ix)+iBx] = Ts1[nTx*(iy*6+ix)]; } } } ozblas-1.2a17/src/cuda/blas/mublas_dot_5_kernel.cu0000644000175200017520000000460513725177602022362 0ustar mukunokimukunoki__global__ void mublas_sum_5_device_s1 ( const size_t n, FP_CTYPE *w, const size_t ldw ) { const size_t iTx = threadIdx.x; const size_t nTx = DOTGEMM_NTX_SMALL;//blockDim.x; size_t i, ix, iy; extern __shared__ FP_CTYPE Ts1[]; FP_CTYPE Tr1[5*5] = {0.}; for (i = iTx; i < n; i += nTx) { #pragma unroll for (ix = 0; ix < 5; ix++) { #pragma unroll for (iy = 0; iy < 5; iy++) Tr1[iy*5+ix] += w[ldw*(iy*5+ix)+i]; } } #pragma unroll for (ix = 0; ix < 5; ix++) { #pragma unroll for (iy = 0; iy < 5; iy++) Ts1[nTx*(iy*5+ix)+iTx] = Tr1[iy*5+ix]; } __syncthreads (); #pragma unroll for (i = nTx/2; i > 0; i >>= 1) { if (iTx < i) { #pragma unroll for (ix = 0; ix < 5; ix++) { #pragma unroll for (iy = 0; iy < 5; iy++) Ts1[nTx*(iy*5+ix)+iTx] += Ts1[nTx*(iy*5+ix)+iTx+i]; } } __syncthreads (); } if (iTx == 0) { #pragma unroll for (ix = 0; ix < 5; ix++) { #pragma unroll for (iy = 0; iy < 5; iy++) w[iy*5+ix] = Ts1[nTx*(iy*5+ix)]; } } } __global__ void mublas_dot_5_device_s1 ( const size_t n, const FP_CTYPE * __restrict__ x, const size_t ldx, const FP_CTYPE * __restrict__ y, const size_t ldy, FP_CTYPE *w, const size_t ldw ) { const size_t iTx = threadIdx.x; const size_t iBx = blockIdx.x; const size_t nBx = gridDim.x; const size_t nTx = DOTGEMM_NTX_SMALL;//blockDim.x; const size_t addrx = iBx * nTx + iTx; size_t i, ix, iy; extern __shared__ FP_CTYPE Ts1[]; FP_CTYPE Xr1[5], Yr1[5]; FP_CTYPE Tr1[5*5] = {0.}; for (i = addrx; i < n; i += nTx * nBx) { #pragma unroll for (ix = 0; ix < 5; ix++) Xr1[ix] = x[ix * ldx + i]; #pragma unroll for (iy = 0; iy < 5; iy++) Yr1[iy] = y[iy * ldy + i]; #pragma unroll for (ix = 0; ix < 5; ix++) { #pragma unroll for (iy = 0; iy < 5; iy++) Tr1[iy*5+ix] += Xr1[ix] * Yr1[iy]; } } #pragma unroll for (ix = 0; ix < 5; ix++) { #pragma unroll for (iy = 0; iy < 5; iy++) Ts1[nTx*(iy*5+ix)+iTx] = Tr1[iy*5+ix]; } __syncthreads (); #pragma unroll for (i = nTx/2; i > 0; i >>= 1) { if (iTx < i) { #pragma unroll for (ix = 0; ix < 5; ix++) { #pragma unroll for (iy = 0; iy < 5; iy++) Ts1[nTx*(iy*5+ix)+iTx] += Ts1[nTx*(iy*5+ix)+iTx+i]; } } __syncthreads (); } if (iTx == 0) { #pragma unroll for (ix = 0; ix < 5; ix++) { #pragma unroll for (iy = 0; iy < 5; iy++) w[ldw*(iy*5+ix)+iBx] = Ts1[nTx*(iy*5+ix)]; } } } ozblas-1.2a17/src/cuda/blas/mublas_dot_4_kernel.cu0000644000175200017520000000457113725177602022363 0ustar mukunokimukunoki__global__ void mublas_sum_4_device_s1 ( const size_t n, FP_CTYPE *w, const size_t ldw ) { const size_t iTx = threadIdx.x; const size_t nTx = DOTGEMM_NTX;//blockDim.x; size_t i, ix, iy; extern __shared__ FP_CTYPE Ts1[]; FP_CTYPE Tr1[4*4] = {0.}; for (i = iTx; i < n; i += nTx) { #pragma unroll for (ix = 0; ix < 4; ix++) { #pragma unroll for (iy = 0; iy < 4; iy++) Tr1[iy*4+ix] += w[ldw*(iy*4+ix)+i]; } } #pragma unroll for (ix = 0; ix < 4; ix++) { #pragma unroll for (iy = 0; iy < 4; iy++) Ts1[nTx*(iy*4+ix)+iTx] = Tr1[iy*4+ix]; } __syncthreads (); #pragma unroll for (i = nTx/2; i > 0; i >>= 1) { if (iTx < i) { #pragma unroll for (ix = 0; ix < 4; ix++) { #pragma unroll for (iy = 0; iy < 4; iy++) Ts1[nTx*(iy*4+ix)+iTx] += Ts1[nTx*(iy*4+ix)+iTx+i]; } } __syncthreads (); } if (iTx == 0) { #pragma unroll for (ix = 0; ix < 4; ix++) { #pragma unroll for (iy = 0; iy < 4; iy++) w[iy*4+ix] = Ts1[nTx*(iy*4+ix)]; } } } __global__ void mublas_dot_4_device_s1 ( const size_t n, const FP_CTYPE * __restrict__ x, const size_t ldx, const FP_CTYPE * __restrict__ y, const size_t ldy, FP_CTYPE *w, const size_t ldw ) { const size_t iTx = threadIdx.x; const size_t iBx = blockIdx.x; const size_t nBx = gridDim.x; const size_t nTx = DOTGEMM_NTX;//blockDim.x; const size_t addrx = iBx * nTx + iTx; size_t i, ix, iy; extern __shared__ FP_CTYPE Ts1[]; FP_CTYPE Xr1[4], Yr1[4]; FP_CTYPE Tr1[4*4] = {0.}; for (i = addrx; i < n; i += nTx * nBx) { #pragma unroll for (ix = 0; ix < 4; ix++) Xr1[ix] = x[ix * ldx + i]; #pragma unroll for (iy = 0; iy < 4; iy++) Yr1[iy] = y[iy * ldy + i]; #pragma unroll for (ix = 0; ix < 4; ix++) { #pragma unroll for (iy = 0; iy < 4; iy++) Tr1[iy*4+ix] += Xr1[ix] * Yr1[iy]; } } #pragma unroll for (ix = 0; ix < 4; ix++) { #pragma unroll for (iy = 0; iy < 4; iy++) Ts1[nTx*(iy*4+ix)+iTx] = Tr1[iy*4+ix]; } __syncthreads (); #pragma unroll for (i = nTx/2; i > 0; i >>= 1) { if (iTx < i) { #pragma unroll for (ix = 0; ix < 4; ix++) { #pragma unroll for (iy = 0; iy < 4; iy++) Ts1[nTx*(iy*4+ix)+iTx] += Ts1[nTx*(iy*4+ix)+iTx+i]; } } __syncthreads (); } if (iTx == 0) { #pragma unroll for (ix = 0; ix < 4; ix++) { #pragma unroll for (iy = 0; iy < 4; iy++) w[ldw*(iy*4+ix)+iBx] = Ts1[nTx*(iy*4+ix)]; } } } ozblas-1.2a17/src/cuda/blas/mublas_dot_kernel.cu0000644000175200017520000000444613725177602022141 0ustar mukunokimukunoki__global__ void MUBLAS_SUM_DEVICE_s1 ( const size_t n, const size_t sx, const size_t sy, FP_CTYPE *w, const size_t ldw ) { const size_t iTx = threadIdx.x; const size_t iBx = blockIdx.x; const size_t nBx = gridDim.x; const size_t nTx = DOTGEMM_NTX;//blockDim.x; const size_t addrx = iBx * nTx + iTx; size_t i, ix, iy; extern __shared__ FP_CTYPE Ts1[]; FP_CTYPE Tr1[MAXNVEC*MAXNVEC] = {0.}; for (i = addrx; i < n; i += nTx * nBx) for (ix = 0; ix < sx; ix++) for (iy = 0; iy < sy; iy++) Tr1[iy*MAXNVEC+ix] += w[ldw*(iy*sx+ix)+i]; for (ix = 0; ix < sx; ix++) for (iy = 0; iy < sy; iy++) Ts1[nTx*(iy*sx+ix)+iTx] = Tr1[iy*MAXNVEC+ix]; __syncthreads (); #pragma unroll for (i = nTx/2; i > 0; i >>= 1) { if (iTx < i) for (ix = 0; ix < sx; ix++) for (iy = 0; iy < sy; iy++) Ts1[nTx*(iy*sx+ix)+iTx] += Ts1[nTx*(iy*sx+ix)+iTx+i]; __syncthreads (); } if (iTx == 0) for (ix = 0; ix < sx; ix++) for (iy = 0; iy < sy; iy++) w[nBx*(iy*sx+ix)+iBx] = Ts1[nTx*(iy*sx+ix)]; } __global__ void MUBLAS_DOT_DEVICE_s1 ( const size_t n, const size_t sx, const size_t sy, const FP_CTYPE * __restrict__ x, const size_t ldx, const FP_CTYPE * __restrict__ y, const size_t ldy, FP_CTYPE *w, const size_t ldw ) { const size_t iTx = threadIdx.x; const size_t iBx = blockIdx.x; const size_t nBx = gridDim.x; const size_t nTx = DOTGEMM_NTX;//blockDim.x; const size_t addrx = iBx * nTx + iTx; size_t i, ix, iy; extern __shared__ FP_CTYPE Ts1[]; FP_CTYPE Xr1[MAXNVEC], Yr1[MAXNVEC]; FP_CTYPE Tr1[MAXNVEC*MAXNVEC] = {0.}; w[addrx] = 0.; for (i = addrx; i < n; i += nTx * nBx) { for (ix = 0; ix < sx; ix++) Xr1[ix] = x[ix * ldx + i]; for (iy = 0; iy < sy; iy++) Yr1[iy] = y[iy * ldy + i]; for (ix = 0; ix < sx; ix++) for (iy = 0; iy < sy; iy++) Tr1[iy*MAXNVEC+ix] += Xr1[ix] * Yr1[iy]; } for (ix = 0; ix < sx; ix++) for (iy = 0; iy < sy; iy++) Ts1[nTx*(iy*sx+ix)+iTx] = Tr1[iy*MAXNVEC+ix]; __syncthreads (); #pragma unroll for (i = nTx/2; i > 0; i >>= 1) { if (iTx < i) for (ix = 0; ix < sx; ix++) for (iy = 0; iy < sy; iy++) Ts1[nTx*(iy*sx+ix)+iTx] += Ts1[nTx*(iy*sx+ix)+iTx+i]; __syncthreads (); } if (iTx == 0) for (ix = 0; ix < sx; ix++) for (iy = 0; iy < sy; iy++) w[ldw*(iy*sx+ix)+iBx] = Ts1[nTx*(iy*sx+ix)]; } ozblas-1.2a17/src/cuda/cuozblas_axpy.cu0000644000175200017520000000254113725177602020404 0ustar mukunokimukunoki#include "cuozblas_common.h" #define NTX_AXPY 512 __global__ void PdevKernel ( const int32_t n, const FP_DTYPE *alpha, const FP_DTYPE * __restrict__ devX, FP_DTYPE *devY ) { const size_t rowid = blockIdx.x * blockDim.x + threadIdx.x; if (rowid < n) devY[rowid] = FMA (alpha[0], devX[rowid], devY[rowid]); } __global__ void PhstKernel ( const int32_t n, const FP_DTYPE alpha, const FP_DTYPE* __restrict__ devX, FP_DTYPE * devY ) { const size_t rowid = blockIdx.x * blockDim.x + threadIdx.x; if (rowid < n) devY[rowid] = FMA (alpha, devX[rowid], devY[rowid]); } __host__ int32_t OzAXPY ( cuozblasHandle_t *oh, const int32_t n, const FP_DTYPE *alpha, const FP_DTYPE *devX, const int32_t incx, FP_DTYPE *devY, const int32_t incy ) { cublasPointerMode_t pmode; cublasGetPointerMode(oh->ch, &pmode); cublasSetPointerMode(oh->ch, CUBLAS_POINTER_MODE_HOST); if (oh->reproModeFlag == 0) { RCuAXPY (oh->ch, n, alpha, devX, incx, devY, incy); return 0; } if (incx != 1 || incy != 1) { printf ("incx and incy are not supported.\n"); return 0; } dim3 threads = dim3 (NTX_AXPY); dim3 grid = dim3 (ceil ((double)n/NTX_AXPY)); if (pmode == CUBLAS_POINTER_MODE_DEVICE) PdevKernel <<< grid, threads >>> (n, alpha, devX, devY); else PhstKernel <<< grid, threads >>> (n, *alpha, devX, devY); cublasSetPointerMode(oh->ch, pmode); return 0; } ozblas-1.2a17/src/cuda/cuozblas_nrm2.cu0000644000175200017520000000202413725177602020275 0ustar mukunokimukunoki#include "cuozblas_common.h" // reproducible but not accurate __host__ int32_t OzNRM2 ( cuozblasHandle_t *oh, const int32_t n, const FP_DTYPE* devX, const int32_t incx, FP_DTYPE *ret ) { counterInit (oh); if (oh->reproModeFlag == 0 && oh->numSplitArraysMax == 1) { RCuNRM2 (oh->ch, n, devX, incx, ret); return 0; } if (incx != 1) { fprintf (stderr, "* incx and incy are not supported.\n"); exit (1); } cublasPointerMode_t pmode; cublasGetPointerMode(oh->ch, &pmode); FP_DTYPE *ptr = (pmode == CUBLAS_POINTER_MODE_HOST) ? (FP_DTYPE*)oh->devWorkCommon : ret; OzDOT (oh, n, devX, incx, devX, incx, ptr); FP_DTYPE hst_ret; cudaMemcpy (&hst_ret, ptr, sizeof(FP_DTYPE), cudaMemcpyDeviceToHost); // ------------------------------ // computation of SQRT (ret) on host // Not accurate but reproducible hst_ret = sqrt (hst_ret); // ------------------------------ if (pmode == CUBLAS_POINTER_MODE_HOST) ret[0] = hst_ret; else cudaMemcpy (ret, &hst_ret, sizeof(FP_DTYPE), cudaMemcpyHostToDevice); return 0; } ozblas-1.2a17/src/cuda/cuozblas_split_sparse.cu0000644000175200017520000001405413725177602022135 0ustar mukunokimukunoki#include "cuozblas_common.h" __forceinline__ __device__ double NextPowTwo (const double p) { return __dsub_rn(scalbn (p, UNIT), __dmul_rn(__dsub_rn(scalbn (1., UNIT), 1), p)); } // ========================================= // SplitMat // ========================================= __global__ void cuozblasFindMaxMatSparseNKernel ( const int32_t m, const FP_CTYPE * __restrict__ devInput, const int32_t * __restrict__ devRowptr, FP_CTYPE *devMax ) { const size_t iBx = blockIdx.x; const size_t iTx = threadIdx.x; const size_t tid = blockDim.x * iBx + iTx; const size_t rowid = tid / 32; const size_t lane = tid % 32; FP_CTYPE max, input, tmp; __shared__ FP_CTYPE shm[512]; if (rowid < m){ max = 0.; for (size_t i = devRowptr[rowid] + lane; i < devRowptr[rowid+1]; i += 32) { input = devInput[i]; if (max < fabs(input)) max = fabs(input); } shm[(iTx/32)*32+lane] = max; __syncthreads (); if (lane == 0) { max = shm[(iTx/32)*32]; #pragma unroll for (size_t i = 1; i < 32; i++) { tmp = shm[(iTx/32)*32+i]; if (max < fabs(tmp)) max = fabs(tmp); } devMax[rowid] = max; } } } __host__ void cuozblasFindMaxMatSparseDevice ( const char trans, const int32_t m, const FP_CTYPE *devInput, const int32_t *devRowptr, FP_CTYPE *devMax ) { int32_t ntx, nbx; dim3 threads, grid; if (trans == 'n' || trans == 'N') { ntx = 512; nbx = ceil (float(m) / (ntx / 32.)); threads = dim3 (ntx); grid = dim3 (nbx); cuozblasFindMaxMatSparseNKernel <<< grid, threads >>> (m, devInput, devRowptr, devMax); } else { fprintf (stderr, "error: Split T is not implemented.\n"); exit (1); } } __global__ void cuozblasSplitMatSparseNKernel ( const int32_t m, const FP_CTYPE * __restrict__ devInput, const int32_t *devRowptr, FP_CTYPE *devOutput, FP_CTYPE *devSplit, FP_CTYPE *devMax ) { const size_t iBx = blockIdx.x; const size_t iTx = threadIdx.x; const size_t tid = blockDim.x * iBx + iTx; const size_t rowid = tid / 32; const size_t lane = tid % 32; FP_CTYPE sigma, max, input, split, tmp; __shared__ FP_CTYPE shm[512]; if (rowid < m){ int32_t dim = devRowptr[rowid+1] - devRowptr[rowid]; int32_t rho = ceil((double)(UNIT+log2((double)dim))/2.); max = devMax[rowid]; sigma = scalbn (1., rho) * NextPowTwo (max); max = 0.; for (size_t i = devRowptr[rowid] + lane; i < devRowptr[rowid+1]; i += 32) { input = devInput[i]; split = SUB (ADD (input, sigma), sigma); input = SUB (input, split); devSplit[i] = split; devOutput[i] = input; if (max < fabs(input)) max = fabs(input); } shm[(iTx/32)*32+lane] = max; __syncthreads (); if (lane == 0) { max = shm[(iTx/32)*32]; #pragma unroll for (size_t i = 1; i < 32; i++) { tmp = shm[(iTx/32)*32+i]; if (max < fabs(tmp)) max = fabs(tmp); } devMax[rowid] = max; } } } __global__ void cuozblasSplitMatSparseEarlyExitNKernel ( const int32_t m, const FP_CTYPE * __restrict__ devInput, const int32_t * __restrict__ devRowptr, FP_CTYPE *devSplit, FP_CTYPE *devMax ) { const size_t iBx = blockIdx.x; const size_t iTx = threadIdx.x; const size_t tid = blockDim.x * iBx + iTx; const size_t rowid = tid / 32; const size_t lane = tid % 32; FP_CTYPE sigma, max, input, split; if (rowid < m){ int32_t dim = devRowptr[rowid+1] - devRowptr[rowid]; int32_t rho = ceil((double)(UNIT+log2((double)dim))/2.); max = devMax[rowid]; sigma = scalbn (1., rho) * NextPowTwo (max); for (size_t i = devRowptr[rowid] + lane; i < devRowptr[rowid+1]; i += 32) { input = devInput[i]; split = SUB (ADD (input, sigma), sigma); input = SUB (input, split); devSplit[i] = split; } } } __host__ void cuozblasSplitMatSparseDevice ( const char trans, const int32_t m, const FP_CTYPE *devInput, // input matrix (devAwrk) const int32_t *devRowptr, FP_CTYPE *devOutput, // output matrix (devAwrk) FP_CTYPE *devSplit, // split matrices (output): this includes NumSplitMax matrices FP_CTYPE *devMax, uint32_t lastFlag ) { size_t ntx, nbx; dim3 threads, grid; /* FP_CTYPE *hstC, *devC; cudaMallocHost ((void **) &hstC, sizeof(FP_CTYPE) * m); cudaMalloc ((void **) &devC, sizeof(FP_CTYPE) * m); */ if (trans == 'n' || trans == 'N') { //PrintMat (m, 1, devMax, m); ntx = 512; nbx = ceil (float(m) / (ntx / 32.)); threads = dim3 (ntx); grid = dim3 (nbx); if (!lastFlag) cuozblasSplitMatSparseNKernel <<< grid, threads >>> (m, devInput, devRowptr, devOutput, devSplit, devMax); else cuozblasSplitMatSparseEarlyExitNKernel <<< grid, threads >>> (m, devInput, devRowptr, devSplit, devMax); } else { fprintf (stderr, "error: Split T is not implemented.\n"); exit (1); } /* cudaMemcpy (hstC, devC, sizeof(FP_CTYPE) * m, cudaMemcpyDeviceToHost); for (int32_t i = 0; i < m; i++) printf ("%d:%a\n", i, hstC[i]); cudaFree (devC); cudaFreeHost (hstC); */ } __host__ void cuozblasSplitMatSparse ( cuozblasHandle_t *oh, const char trans, const int32_t m, const FP_CTYPE *devInput, const int32_t *devRowptr, FP_CTYPE *devSplit, FP_CTYPE *devMax, const int32_t lds, uint32_t &numSplitArrays, uint32_t &lastFlag ) { uint32_t numSplitArraysMax = numSplitArrays; uint32_t i = 1; FP_CTYPE check; // FindMax^(0) cuozblasFindMaxMatSparseDevice (trans, m, devInput, devRowptr, devMax); // Split^(0) & FindMax^(1) lastFlag = (oh->reproModeFlag && numSplitArraysMax == 1) ? 1 : 0; cuozblasSplitMatSparseDevice (trans, m, devInput, devRowptr, &devSplit[lds], &devSplit[0], devMax, lastFlag); while (i < numSplitArraysMax) { // Check if (numSplitArraysMax-1 == i && !oh->reproModeFlag) { i++; break;} // when repromode=0 CuASUM (oh->ch, m, devMax, 1, &check); if (fabs(check) == 0) break; // Split^(i) & FindMax^(i+1) lastFlag = (oh->reproModeFlag && numSplitArraysMax-1 == i) ? 1 : 0; cuozblasSplitMatSparseDevice (trans, m, &devSplit[lds*i], devRowptr, &devSplit[lds*(i+1)], &devSplit[lds*i], devMax, lastFlag); i++; } numSplitArrays = i; if (lastFlag == 1 && oh->numSplitArraysMax == 0) { fprintf (stderr, "split error: out of memory (the result may be not correct rounding)\n"); exit (1); } } #include "cuozblas_split_sparse_shift.cu" ozblas-1.2a17/src/cuda/cuozblas_dot.cu0000644000175200017520000001305013725177602020206 0ustar mukunokimukunoki#include "cuozblas_common.h" #include "blas/mublas_dot.cu" // ========================================= // InitVec // ========================================= __global__ void cuozblasInitVecKernel ( const size_t n, int64_t *devA ) { const size_t iBx = blockIdx.x; const size_t nTx = blockDim.x; const size_t iTx = threadIdx.x; const size_t addrx = iBx * nTx + iTx; int64_t *addrA = &devA [addrx]; if (addrx < n) addrA[0] = 0; } __host__ void cuozblasInitVecDevice ( const size_t n, int64_t *devA ) { dim3 threads, grid; int32_t ntx, nbx; ntx = 128; nbx = ceil (float(n) / ntx); threads = dim3 (ntx); grid = dim3 (nbx); cuozblasInitVecKernel <<< grid, threads >>> (n, devA); } __host__ int32_t OzDOT ( cuozblasHandle_t *oh, const int32_t n, const FP_DTYPE *devA, const int32_t incx, const FP_DTYPE *devB, const int32_t incy, FP_DTYPE *ret ) { counterInit (oh); if (oh->reproModeFlag == 0 && oh->numSplitArraysMax == 1) { double t0 = timer(); RCuDOT (oh->ch, n, devA, incx, devB, incy, ret); oh->t_DOT_NRM2_total += timer() - t0; return 0; } if (incx != 1 || incy != 1 ) { fprintf (stderr, "* incx and incy are not supported.\n"); exit (1); } cublasPointerMode_t pmode; cublasGetPointerMode(oh->ch, &pmode); FP_DTYPE fone = 1., fzero = 0.; FP_CTYPE cfone = 1., cfzero = 0.; double t1, t0 = timer(); uint32_t numSplitArraysMax = (oh->numSplitArraysMax > 0) ? oh->numSplitArraysMax : NumSplitDefaultMax; int32_t ldas, ldbs, ldcs; FP_CTYPE *devAmax, *devBmax; FP_CTYPE *devASplit, *devBSplit, *devCSplit; if (oh->numSplitArraysMax > 0) numSplitArraysMax = oh->numSplitArraysMax; cuozblasMallocSplitVec (oh, n, numSplitArraysMax, &devASplit, &devAmax, ldas); cuozblasMallocSplitVec (oh, n, numSplitArraysMax, &devBSplit, &devBmax, ldbs); cuozblasMallocResultVec (oh, 1, numSplitArraysMax, &devCSplit, ldcs); #if defined (PREC_SD) int32_t ldat, ldbt; FP_CTYPE *devATmp, *devBTmp; cuozblasSDMallocTmpVec (oh, n, &devATmp, ldat); cuozblasSDMallocTmpVec (oh, n, &devBTmp, ldbt); cuozblasSDCopyMat (n, 1, devA, n, devATmp, ldat); cuozblasSDCopyMat (n, 1, devB, n, devBTmp, ldbt); #endif memCheck (oh); cuozblasInitVecDevice (ldcs * numSplitArraysMax * numSplitArraysMax, (int64_t*)devCSplit); // Split of A ----------------------------------- t1 = timer(); oh->numSplitArraysA = numSplitArraysMax; #if defined (PREC_SD) cuozblasSplitMat (oh, 'r', n, 1, devATmp, n, devASplit, devAmax, ldas, oh->numSplitArraysA, oh->lastFlagA); #else cuozblasSplitMat (oh, 'r', n, 1, devA, n, devASplit, devAmax, ldas, oh->numSplitArraysA, oh->lastFlagA); #endif oh->t_SplitA = timer() - t1; // Split of B ----------------------------------- t1 = timer(); oh->numSplitArraysB = numSplitArraysMax; #if defined (PREC_SD) cuozblasSplitMat (oh, 'r', n, 1, devBTmp, n, devBSplit, devBmax, ldbs, oh->numSplitArraysB, oh->lastFlagB); #else cuozblasSplitMat (oh, 'r', n, 1, devB, n, devBSplit, devBmax, ldbs, oh->numSplitArraysB, oh->lastFlagB); #endif oh->t_SplitB = timer() - t1; // Compute -------------------------------------- t1 = timer(); uint32_t ia, ib; uint32_t maxlevel = (oh->numSplitArraysA-1) + (oh->numSplitArraysB-1); if (oh->fastModeFlag) { if (oh->numSplitArraysMax == 0) maxlevel = min (oh->numSplitArraysA-1, oh->numSplitArraysB-1); else maxlevel = oh->numSplitArraysMax-1; } if (oh->useGemmFlag) { cublasSetPointerMode(oh->ch, CUBLAS_POINTER_MODE_HOST); if (oh->useMyGemmFlag) { FP_CTYPE *devW = (FP_CTYPE*)(oh->devWork + oh->memAddr); oh->memAddr += sizeof(FP_CTYPE) * oh->numSplitArraysA * oh->numSplitArraysB * SPLIT_VEC_NBX; memCheck (oh); MuMDOT (oh->ch, n, oh->numSplitArraysA, oh->numSplitArraysB, devASplit, ldas, devBSplit, ldbs, devCSplit, oh->numSplitArraysA, devW); } else { CuGEMM (oh->ch, CUBLAS_OP_T, CUBLAS_OP_N, oh->numSplitArraysA, oh->numSplitArraysB, n, &cfone, devASplit, ldas, devBSplit, ldbs, &cfzero, devCSplit, oh->numSplitArraysA); } oh->numSplitArraysC = oh->numSplitArraysA * oh->numSplitArraysB; // to ensure the consistency with the computation result by GEMM when FS if (oh->fastModeFlag) { FP_CTYPE zero = 0.; for (ia = 0; ia < oh->numSplitArraysA; ia++) { for (ib = 0; ib < oh->numSplitArraysB; ib++) { if (ia + ib > maxlevel) cudaMemcpy (devCSplit+oh->numSplitArraysA*ib+ia, &zero, sizeof(FP_CTYPE), cudaMemcpyHostToDevice); } } } } else { cublasSetPointerMode(oh->ch, CUBLAS_POINTER_MODE_DEVICE); uint32_t ic = 0; FP_CTYPE *ptrA, *ptrB, *ptrC; for (ia = 0; ia < oh->numSplitArraysA; ia++) { for (ib = 0; ib < oh->numSplitArraysB; ib++) { if (ia + ib <= maxlevel) { ptrA = devASplit+ldas*ia; ptrB = devBSplit+ldbs*ib; ptrC = devCSplit+ic; CuDOT (oh->ch, n, ptrA, 1, ptrB, 1, ptrC); ic++; } } } oh->numSplitArraysC = ic; } oh->t_comp = timer() - t1; // Sum ----------------------------------------- t1 = timer(); FP_DTYPE *ptr = (pmode == CUBLAS_POINTER_MODE_HOST) ? (FP_DTYPE*)oh->devWorkCommon : ret; cuozblasSumMat (1, 1, devCSplit, 1, 1, oh->numSplitArraysC, ptr, 1, &fone, &fzero, pmode, oh->sumModeFlag); if (pmode == CUBLAS_POINTER_MODE_HOST) cudaMemcpy (ret, ptr, sizeof(FP_DTYPE), cudaMemcpyDeviceToHost); oh->t_sum = timer() - t1; oh->t_total = timer() - t0; // for CG, time // ================================= oh->t_SplitMat_total += 0.; oh->t_SplitVec_total += oh->t_SplitA + oh->t_SplitB; oh->t_Sum_total += oh->t_sum; oh->t_AXPY_SCAL_total += 0.; oh->t_DOT_NRM2_total += oh->t_comp; oh->t_SpMV_SpMM_total += 0.; // ================================= cublasSetPointerMode(oh->ch, pmode); return 0; } ozblas-1.2a17/src/cuda/cuozblas_gemm.cu0000644000175200017520000002346113725177602020354 0ustar mukunokimukunoki#include "cuozblas_common.h" #if !defined (PREC_SD) __host__ int32_t OzGEMM ( cuozblasHandle_t *oh, const cublasOperation_t tranA, const cublasOperation_t tranB, const int32_t m, const int32_t n, const int32_t k, const FP_DTYPE *alpha, const FP_DTYPE *devA, const int32_t lda, const FP_DTYPE *devB, const int32_t ldb, const FP_DTYPE *beta, FP_DTYPE *devC, const int32_t ldc ) { counterInit (oh); if (oh->reproModeFlag == 0 && oh->numSplitArraysMax == 1) { RCuGEMM (oh->ch, tranA, tranB, m, n, k, alpha, devA, lda, devB, ldb, beta, devC, ldc); return 0; } cublasPointerMode_t pmode; cublasGetPointerMode(oh->ch, &pmode); cublasSetPointerMode(oh->ch, CUBLAS_POINTER_MODE_HOST); FP_CTYPE fone = 1., fzero = 0.; FP_CTYPE t1, t0 = timer(); uint32_t numSplitArraysMax = (oh->numSplitArraysMax > 0) ? oh->numSplitArraysMax : NumSplitDefaultMax; int32_t ldas, ldbs, ldcs; FP_CTYPE *devAmax, *devBmax; FP_CTYPE *devASplit, *devBSplit, *devCSplit; int32_t mbk = m; int32_t nbk = n; size_t memAddrTmp = oh->memAddr; while (mbk > 0) { if (tranA == CUBLAS_OP_N) cuozblasMallocSplitMat (oh, mbk, k, numSplitArraysMax, &devASplit, &devAmax, ldas); else cuozblasMallocSplitMat (oh, k, mbk, numSplitArraysMax, &devASplit, &devAmax, ldas); if (tranB == CUBLAS_OP_N) cuozblasMallocSplitMat (oh, k, nbk, numSplitArraysMax, &devBSplit, &devBmax, ldbs); else cuozblasMallocSplitMat (oh, nbk, k, numSplitArraysMax, &devBSplit, &devBmax, ldbs); cuozblasMallocResultMat (oh, mbk, nbk, numSplitArraysMax, &devCSplit, ldcs); if (!memCheck (oh)) break; oh->memAddr = memAddrTmp; mbk = ceil (mbk / 2.); nbk = ceil (nbk / 2.); } FP_CTYPE **hstAptr, **hstBptr, **hstCptr; FP_CTYPE **devAptr, **devBptr, **devCptr; if (oh->useBatchedGemmFlag) { size_t memorysize = 0; hstAptr = (FP_CTYPE**)(oh->hstBatchAddr); memorysize += sizeof(FP_CTYPE*) * numSplitArraysMax * numSplitArraysMax; hstBptr = (FP_CTYPE**)(oh->hstBatchAddr + memorysize); memorysize += sizeof(FP_CTYPE*) * numSplitArraysMax * numSplitArraysMax; hstCptr = (FP_CTYPE**)(oh->hstBatchAddr + memorysize); memorysize = 0; devAptr = (FP_CTYPE**)(oh->devBatchAddr); memorysize += sizeof(FP_CTYPE*) * numSplitArraysMax * numSplitArraysMax; devBptr = (FP_CTYPE**)(oh->devBatchAddr + memorysize); memorysize += sizeof(FP_CTYPE*) * numSplitArraysMax * numSplitArraysMax; devCptr = (FP_CTYPE**)(oh->devBatchAddr + memorysize); } // --------------------------------------------- // Dense to Sparse cusparseDirection_t dirA = CUSPARSE_DIRECTION_ROW; cusparseMatDescr_t descrA; cusparseCreateMatDescr (&descrA); cusparseSetMatType (descrA, CUSPARSE_MATRIX_TYPE_GENERAL); cusparseSetMatIndexBase (descrA, CUSPARSE_INDEX_BASE_ZERO); cusparseSetPointerMode (oh->csh, CUSPARSE_POINTER_MODE_HOST); // --------------------------------------------- for (int32_t im = 0; im < ceil((float)m/mbk); im++) { int32_t mbk_ = (m-mbk*im >= mbk) ? mbk : m-mbk*im; // Split of A ----------------------------------- t1 = timer(); oh->numSplitArraysA = numSplitArraysMax; if (tranA == CUBLAS_OP_N) cuozblasSplitMat (oh, 'r', mbk_, k, devA+im*mbk, lda, devASplit, devAmax, ldas, oh->numSplitArraysA, oh->lastFlagA); else cuozblasSplitMat (oh, 'c', k, mbk_, devA+im*mbk, lda, devASplit, devAmax, ldas, oh->numSplitArraysA, oh->lastFlagA); // --------------------------------------------- // Dense to Sparse // * this is experimental: only for GEMM-NN // * it can be activated with sparseThreshold > 0 // * (batched GEMM cannot be used together) uint32_t ia; int32_t nnzA[NumSplitDefaultMax] = {0}; if (oh->useBatchedGemmFlag == 0 && oh->sparseThreshold > 0 && tranA == CUBLAS_OP_N && tranB == CUBLAS_OP_N) { size_t memsizeCsrConvWork1 = sizeof(FP_CTYPE) * getPitchSize(mbk_ * k) + sizeof(int32_t) * (getPitchSize(mbk_ * k) + getPitchSize(mbk_ + 1) + mbk_); oh->memAddr += memsizeCsrConvWork1; if (memCheck (oh)) { fprintf (stderr, "* dense->sparse: memory allocation error (requested: %1.3e GB, workSize: %1.3e GB)\n", 1.e-9*oh->memAddr, 1.e-9*oh->workSizeBytes); exit (1); } oh->memAddr -= memsizeCsrConvWork1; FP_CTYPE *devCsrValA = (FP_CTYPE*)(oh->devWork + oh->memAddr); int32_t *devCsrColIndA = (int32_t*)(devCsrValA + getPitchSize (mbk_ * k)); int32_t *devCsrRowPtrA = devCsrColIndA + getPitchSize (mbk_ * k); int32_t *devCsrNnzPerRow = devCsrRowPtrA + getPitchSize (mbk_ + 1); // here, oh->memAddr is recovered before the above allocation for (ia = 0; ia < oh->numSplitArraysA; ia++) { if (tranA == CUBLAS_OP_N) { FP_CTYPE *ptrA = devASplit+ldas*k*ia; CuNNZ (oh->csh, dirA, mbk_, k, descrA, ptrA, ldas, devCsrNnzPerRow, &nnzA[ia]); float nnzPercentage = 100. * nnzA[ia] / (mbk_ * k); size_t memsizeCsrConvWork2 = sizeof(FP_CTYPE) * getPitchSize(nnzA[ia]) + sizeof(int32_t) * (getPitchSize(nnzA[ia]) + mbk_+1); // if the size is fit into devASplit[ia] //printf ("%f[%] / %f[%]\n", nnzPercentage, oh->sparseThreshold); if (nnzPercentage <= oh->sparseThreshold && memsizeCsrConvWork2 <= (sizeof(FP_CTYPE) * mbk_ * ldas)) { //printf ("conv.\n"); CuDENSE2CSR (oh->csh, mbk_, k, descrA, ptrA, ldas, devCsrNnzPerRow, devCsrValA, devCsrRowPtrA, devCsrColIndA); FP_CTYPE *devAval = ptrA; int32_t *devAcolind = (int32_t*)(devAval + getPitchSize (nnzA[ia])); int32_t *devArowptr = devAcolind + getPitchSize (nnzA[ia]); // overwrite to devASplit[ia] cudaMemcpy (devAval, devCsrValA, sizeof(FP_CTYPE)*(nnzA[ia]), cudaMemcpyDeviceToDevice); cudaMemcpy (devAcolind, devCsrColIndA, sizeof(int32_t)*nnzA[ia], cudaMemcpyDeviceToDevice); cudaMemcpy (devArowptr, devCsrRowPtrA, sizeof(int32_t)*(mbk_+1), cudaMemcpyDeviceToDevice); } else { // otherwise ignore the conversion nnzA[ia] = 0; } } } } // --------------------------------------------- oh->t_SplitA += timer() - t1; for (int32_t in = 0; in < ceil((float)n/nbk); in++) { int32_t nbk_ = (n-nbk*in >= nbk) ? nbk : n-nbk*in; // Split of B ----------------------------------- t1 = timer(); oh->numSplitArraysB = numSplitArraysMax; if (tranB == CUBLAS_OP_N) cuozblasSplitMat (oh, 'c', k, nbk_, devB+in*nbk*ldb, ldb, devBSplit, devBmax, ldbs, oh->numSplitArraysB, oh->lastFlagB); else cuozblasSplitMat (oh, 'r', nbk_, k, devB+in*nbk*ldb, ldb, devBSplit, devBmax, ldbs, oh->numSplitArraysB, oh->lastFlagB); oh->t_SplitB += timer() - t1; // Compute -------------------------------------- uint32_t ia, ib, ic; uint32_t maxlevel = (oh->numSplitArraysA-1) + (oh->numSplitArraysB-1); if (oh->fastModeFlag) { if (oh->numSplitArraysMax == 0) maxlevel = min (oh->numSplitArraysA-1, oh->numSplitArraysB-1); else maxlevel = oh->numSplitArraysMax-1; } if (oh->numSplitArraysMax == 1) { t1 = timer(); CuGEMM (oh->ch, tranA, tranB, mbk_, nbk_, k, &fone, devASplit, ldas, devBSplit, ldbs, &fzero, devCSplit, ldcs); oh->numSplitArraysC = 1; } else { ic = 0; if (oh->useBatchedGemmFlag) { for (ia = 0; ia < oh->numSplitArraysA; ia++) { for (ib = 0; ib < oh->numSplitArraysB; ib++) { if (ia + ib <= maxlevel) { if (tranA == CUBLAS_OP_N) hstAptr[ic] = devASplit+ldas*k*ia; else hstAptr[ic] = devASplit+ldas*mbk*ia; if (tranB == CUBLAS_OP_N) hstBptr[ic] = devBSplit+ldbs*nbk*ib; else hstBptr[ic] = devBSplit+ldbs*k*ib; hstCptr[ic] = devCSplit+ldcs*nbk*ic; ic++; } } } oh->numSplitArraysC = ic; cudaMemcpy(devAptr, hstAptr, sizeof(FP_CTYPE*) * oh->numSplitArraysC, cudaMemcpyHostToDevice); cudaMemcpy(devBptr, hstBptr, sizeof(FP_CTYPE*) * oh->numSplitArraysC, cudaMemcpyHostToDevice); cudaMemcpy(devCptr, hstCptr, sizeof(FP_CTYPE*) * oh->numSplitArraysC, cudaMemcpyHostToDevice); t1 = timer(); CuGEMMBatched (oh->ch, tranA, tranB, mbk_, nbk_, k, &fone, (const FP_CTYPE**)devAptr, ldas, (const FP_CTYPE**)devBptr, ldbs, &fzero, (FP_CTYPE**)devCptr, ldcs, oh->numSplitArraysC); } else { t1 = timer(); FP_CTYPE *ptrA, *ptrB, *ptrC; for (ia = 0; ia < oh->numSplitArraysA; ia++) { for (ib = 0; ib < oh->numSplitArraysB; ib++) { if (ia + ib <= maxlevel) { if (tranA == CUBLAS_OP_N) ptrA = devASplit+ldas*k*ia; else ptrA = devASplit+ldas*mbk*ia; if (tranB == CUBLAS_OP_N) ptrB = devBSplit+ldbs*nbk*ib; else ptrB = devBSplit+ldbs*k*ib; ptrC = devCSplit+ldcs*nbk*ic; // sparse ------------------------- if (oh->useBatchedGemmFlag == 0 && oh->sparseThreshold > 0 && tranA == CUBLAS_OP_N && tranB == CUBLAS_OP_N) { if (nnzA[ia] > 0) { cusparseOperation_t tranA_ = CUSPARSE_OPERATION_NON_TRANSPOSE; FP_CTYPE *devAval = ptrA; int32_t *devAcolind = (int32_t*)(devAval + getPitchSize (nnzA[ia])); int32_t *devArowptr = devAcolind + getPitchSize (nnzA[ia]); CuCSRMM (oh->csh, tranA_, mbk_, nbk_, k, nnzA[ia], &fone, descrA, devAval, devArowptr, devAcolind, ptrB, ldbs, &fzero, ptrC, ldcs); } else { CuGEMM (oh->ch, tranA, tranB, mbk_, nbk_, k, &fone, ptrA, ldas, ptrB, ldbs, &fzero, ptrC, ldcs); } } else { CuGEMM (oh->ch, tranA, tranB, mbk_, nbk_, k, &fone, ptrA, ldas, ptrB, ldbs, &fzero, ptrC, ldcs); } // sparse ------------------------- ic++; } } } oh->numSplitArraysC = ic; } } oh->t_comp += timer() - t1; // Sum ----------------------------------------- t1 = timer(); cuozblasSumMat (mbk_, nbk_, devCSplit, ldcs*nbk, ldcs, oh->numSplitArraysC, &devC[ldc*(in*nbk)+im*mbk], ldc, alpha, beta, pmode, oh->sumModeFlag); oh->t_sum += timer() - t1; } } oh->t_total = timer() - t0; // printf ("ozblas (%d)\n", oh->numSplitArraysC); cublasSetPointerMode(oh->ch, pmode); return 0; } #endif ozblas-1.2a17/src/cuda/cuozblas_gemv.cu0000644000175200017520000001266113725177602020365 0ustar mukunokimukunoki#include "cuozblas_common.h" __host__ int32_t OzGEMV ( cuozblasHandle_t *oh, const cublasOperation_t tranA, const int32_t m, const int32_t n, const FP_DTYPE *alpha, const FP_DTYPE *devA, const int32_t lda, const FP_DTYPE *devB, const int32_t incx, const FP_DTYPE *beta, FP_DTYPE *devC, const int32_t incy ) { counterInit (oh); if (oh->reproModeFlag == 0 && oh->numSplitArraysMax == 1) { RCuGEMV (oh->ch, tranA, m, n, alpha, devA, lda, devB, incx, beta, devC, incy); return 0; } if (incx != 1 || incy != 1 ) { fprintf (stderr, "* incx and incy are not supported.\n"); exit (1); } cublasPointerMode_t pmode; cublasGetPointerMode(oh->ch, &pmode); cublasSetPointerMode(oh->ch, CUBLAS_POINTER_MODE_HOST); FP_CTYPE fone = 1., fzero = 0.; double t1, t0 = timer(); uint32_t numSplitArraysMax = (oh->numSplitArraysMax > 0) ? oh->numSplitArraysMax : NumSplitDefaultMax; int32_t ldas, ldbs, ldcs; FP_CTYPE *devASplit; FP_CTYPE *devAmax, *devBmax; FP_CTYPE *devBSplit, *devCSplit; int32_t mbk = m; size_t memAddrTmp = oh->memAddr; #if defined (PREC_SD) int32_t ldat, ldbt; FP_CTYPE *devATmp, *devBTmp; #endif while (mbk > 0) { if (tranA == CUBLAS_OP_N){ cuozblasMallocSplitMat (oh, mbk, n, numSplitArraysMax, &devASplit, &devAmax, ldas); cuozblasMallocSplitVec (oh, n, numSplitArraysMax, &devBSplit, &devBmax, ldbs); cuozblasMallocResultVec (oh, mbk, numSplitArraysMax, &devCSplit, ldcs); } else { cuozblasMallocSplitMat (oh, n, mbk, numSplitArraysMax, &devASplit, &devAmax, ldas); cuozblasMallocSplitVec (oh, mbk, numSplitArraysMax, &devBSplit, &devBmax, ldbs); cuozblasMallocResultVec (oh, n, numSplitArraysMax, &devCSplit, ldcs); } #if defined (PREC_SD) if (tranA == CUBLAS_OP_N){ cuozblasSDMallocTmpMat (oh, mbk, n, &devATmp, ldat); cuozblasSDMallocTmpMat (oh, n, 1, &devBTmp, ldbt); cuozblasSDCopyMat (mbk, n, devA, lda, devATmp, ldat); cuozblasSDCopyMat (n, 1, devB, n, devBTmp, ldbt); } else { cuozblasSDMallocTmpMat (oh, n, mbk, &devATmp, ldat); cuozblasSDMallocTmpMat (oh, mbk, 1, &devBTmp, ldbt); cuozblasSDCopyMat (n, mbk, devA, lda, devATmp, ldat); cuozblasSDCopyMat (mbk, 1, devB, mbk, devBTmp, ldbt); } #endif if (!memCheck (oh)) break; oh->memAddr = memAddrTmp; mbk = ceil (mbk / 2.); } for (int32_t im = 0; im < ceil((float)m/mbk); im++) { int32_t mbk_ = (m-mbk*im >= mbk) ? mbk : m-mbk*im; // Split of A ----------------------------------- t1 = timer(); oh->numSplitArraysA = numSplitArraysMax; #if defined (PREC_SD) if (tranA == CUBLAS_OP_N) cuozblasSplitMat (oh, 'r', mbk_, n, devATmp+im*mbk, ldat, devASplit, devAmax, ldas, oh->numSplitArraysA, oh->lastFlagA); else cuozblasSplitMat (oh, 'c', mbk_, n, devATmp+im*mbk, ldat, devASplit, devAmax, ldas, oh->numSplitArraysA, oh->lastFlagA); #else if (tranA == CUBLAS_OP_N) cuozblasSplitMat (oh, 'r', mbk_, n, devA+im*mbk, lda, devASplit, devAmax, ldas, oh->numSplitArraysA, oh->lastFlagA); else cuozblasSplitMat (oh, 'c', mbk_, n, devA+im*mbk, lda, devASplit, devAmax, ldas, oh->numSplitArraysA, oh->lastFlagA); #endif oh->t_SplitA += timer() - t1; // Split of B ----------------------------------- t1 = timer(); oh->numSplitArraysB = numSplitArraysMax; #if defined (PREC_SD) if (tranA == CUBLAS_OP_N) cuozblasSplitMat (oh, 'r', n, 1, devBTmp, n, devBSplit, devBmax, ldbs, oh->numSplitArraysB, oh->lastFlagB); else cuozblasSplitMat (oh, 'r', mbk, 1, devBTmp, mbk, devBSplit, devBmax, ldbs, oh->numSplitArraysB, oh->lastFlagB); #else if (tranA == CUBLAS_OP_N) cuozblasSplitMat (oh, 'r', n, 1, devB, n, devBSplit, devBmax, ldbs, oh->numSplitArraysB, oh->lastFlagB); else cuozblasSplitMat (oh, 'r', mbk, 1, devB, mbk, devBSplit, devBmax, ldbs, oh->numSplitArraysB, oh->lastFlagB); #endif oh->t_SplitB += timer() - t1; // Compute -------------------------------------- t1 = timer(); uint32_t numB, ia, ic = 0; uint32_t maxlevel = (oh->numSplitArraysA-1) + (oh->numSplitArraysB-1); if (oh->fastModeFlag) { if (oh->numSplitArraysMax == 0) maxlevel = min (oh->numSplitArraysA-1, oh->numSplitArraysB-1); else maxlevel = oh->numSplitArraysMax-1; } FP_CTYPE *ptrA, *ptrB, *ptrC; ptrC = devCSplit; for (ia = 0; ia < min (maxlevel+1, oh->numSplitArraysA); ia++) { ptrA = devASplit+ldas*n*ia; if (oh->useGemmFlag) { numB = min (oh->numSplitArraysB, maxlevel+1 - ia); ptrB = devBSplit; if (tranA == CUBLAS_OP_N) CuGEMM (oh->ch, tranA, CUBLAS_OP_N, mbk_, numB, n, &fone, ptrA, ldas, ptrB, ldbs, &fzero, ptrC, ldcs); else CuGEMM (oh->ch, tranA, CUBLAS_OP_N, n, numB, mbk_, &fone, ptrA, ldas, ptrB, ldbs, &fzero, ptrC, ldcs); ptrC += ldcs*numB; ic += numB; } else { for (uint32_t ib = 0; ib < oh->numSplitArraysB; ib++) { if (ia + ib <= maxlevel) { ptrB = devBSplit+ldbs*ib; ptrC = devCSplit+ldcs*ic; CuGEMV (oh->ch, tranA, mbk_, n, &fone, ptrA, ldas, ptrB, 1, &fzero, ptrC, 1); ic++; } } } } oh->numSplitArraysC = ic; oh->t_comp += timer() - t1; // Sum ----------------------------------------- t1 = timer(); if (tranA == CUBLAS_OP_N) cuozblasSumMat (mbk_, 1, devCSplit, ldcs, ldcs, oh->numSplitArraysC, &devC[im*mbk], ldcs, alpha, beta, pmode, oh->sumModeFlag); else cuozblasSumMat (n, 1, devCSplit, ldcs, ldcs, oh->numSplitArraysC, &devC[im*mbk], ldcs, alpha, beta, pmode, oh->sumModeFlag); oh->t_sum += timer() - t1; } oh->t_total = timer() - t0; cublasSetPointerMode(oh->ch, pmode); return 0; } ozblas-1.2a17/src/cuda/cuozblas_auxiliary.cu0000644000175200017520000001365713725177602021444 0ustar mukunokimukunoki#include "cuozblas_common.h" // ========================================= // Print floating-point value with bit representation // ========================================= typedef union{ FP_CTYPE d; uint64_t i; } d_and_i; void printBits (FP_CTYPE val) { d_and_i di; di.d = val; // sign printf ("%u", (di.i >> 63) & 1); printf ("|"); // exponent for (int i = 62; i >= 62-10; i--) printf ("%u", (di.i >> i) & 1); printf ("|"); // fraction for (int i = 62-11; i >= 0; i--) printf ("%u", (di.i >> i) & 1); printf (" : "); printf ("%+1.18e", val); } // ========================================= // PrintMat (for debugging) // ========================================= __host__ void PrintMat ( const int32_t m, const int32_t n, const FP_CTYPE *devC, const int32_t ldd ) { FP_CTYPE *hstC, tmp; int32_t i, j, ldh; ldh = m; cudaMallocHost ((void **) &hstC, sizeof(FP_CTYPE) * ldh * n); cudaMemcpy2D (hstC, sizeof(FP_CTYPE) * ldh, devC, sizeof(FP_CTYPE) * ldd, sizeof(FP_CTYPE) * m, n, cudaMemcpyDeviceToHost); printf ("\n"); for (j = 0; j < n; j++) { for (i = 0; i < m; i++) { tmp = hstC[j*ldh+i]; if (tmp != 0) { printf ("[%2d,%2d] ", j, i); printBits (tmp); printf ("\n"); } } } printf ("\n"); cudaFreeHost (hstC); } __host__ void PrintMatInt ( const int32_t m, const int32_t n, const int32_t *devC, const int32_t ldd ) { int32_t *hstC, tmp; int32_t i, j, ldh; ldh = m; cudaMallocHost ((void **) &hstC, sizeof(int32_t) * ldh * n); cudaMemcpy2D (hstC, sizeof(int32_t) * ldh, devC, sizeof(int32_t) * ldd, sizeof(int32_t) * m, n, cudaMemcpyDeviceToHost); printf ("\n"); for (j = 0; j < n; j++) { for (i = 0; i < m; i++) { tmp = hstC[j*ldh+i]; // if (tmp != 0) { printf ("[%2d,%2d] %d", j, i, tmp); printf ("\n"); // } } } printf ("\n"); cudaFreeHost (hstC); } __host__ void cuozblasPrintVec ( const int32_t n, const FP_DTYPE *devC ) { FP_DTYPE *hstC; cudaMallocHost ((void **) &hstC, sizeof(FP_DTYPE) * n); cudaMemcpy (hstC, devC, sizeof(FP_CTYPE) * n, cudaMemcpyDeviceToHost); for (int32_t i = 0; i < n; i++) printf ("%1.18e\n", hstC[i]); cudaFreeHost (hstC); } // ========================================= // Matrix Allocation // ========================================= __host__ void cuozblasMallocSplitVec ( cuozblasHandle_t *oh, const int32_t n, const uint32_t numSplitArraysMax, FP_CTYPE **devSplit, FP_CTYPE **devMax, int32_t &lds ) { // oh->workSizeBytes; // size of work memory [Bytes] // oh->devWork; // the start point of the work memory // devXsplit starts from the end of devXsplit0 lds = getPitchSize (n); devSplit[0] = (FP_CTYPE*)(oh->devWork + oh->memAddr); oh->memAddr += sizeof(FP_CTYPE) * lds * numSplitArraysMax; size_t ldv = getPitchSize(SPLIT_VEC_NBX+1); // '+1': devMax needs work areas staring from [getPitchSize(1)] devMax[0] = (FP_CTYPE*)(oh->devWork + oh->memAddr); oh->memAddr += sizeof(FP_CTYPE) * ldv; } #if defined (PREC_SD) __host__ void cuozblasSDMallocTmpVec ( cuozblasHandle_t *oh, const int32_t n, FP_CTYPE **devTmp, int32_t &lds ) { lds = getPitchSize (n); devTmp[0] = (FP_CTYPE*)(oh->devWork + oh->memAddr); oh->memAddr += sizeof(FP_CTYPE) * lds; } __host__ void cuozblasSDMallocTmpMat ( cuozblasHandle_t *oh, const int32_t m, const int32_t n, FP_CTYPE **devTmp, int32_t &lds ) { lds = getPitchSize (m); devTmp[0] = (FP_CTYPE*)(oh->devWork + oh->memAddr); oh->memAddr += sizeof(FP_CTYPE) * lds * n; // +1 is for output (work) } __global__ void cuozblasSDCopyMatKernel ( const int32_t m, const int32_t n, const FP_DTYPE *devA, const int32_t lda, FP_CTYPE *devB, const int32_t ldb ) { const size_t iBx = blockIdx.x; const size_t iBy = blockIdx.y; const size_t nTx = blockDim.x; const size_t nTy = blockDim.y; const size_t iTx = threadIdx.x; const size_t iTy = threadIdx.y; const size_t addrx = iBx * nTx + iTx; const size_t addry = iBy * nTy + iTy; if (addrx < m && addry < n) devB[addry*ldb+addrx] = devA[addry*lda+addrx]; } __host__ void cuozblasSDCopyMat ( const int32_t m, const int32_t n, const FP_DTYPE *devA, const int32_t lda, FP_CTYPE *devB, const int32_t ldb ) { dim3 threads, grid; int32_t ntx, nty, nbx, nby; ntx = 256; nty = 2; nbx = ceil (float(m) / ntx); nby = ceil (float(n) / nty); threads = dim3 (ntx, nty); grid = dim3 (nbx, nby); cuozblasSDCopyMatKernel <<< grid, threads >>> (m, n, devA, lda, devB, ldb); } #endif __host__ void cuozblasMallocResultVec ( cuozblasHandle_t *oh, const int32_t n, const uint32_t numSplitArraysMax, FP_CTYPE **devSplit, int32_t &lds ) { lds = getPitchSize (n); devSplit[0] = (FP_CTYPE*)(oh->devWork + oh->memAddr); oh->memAddr += sizeof(FP_CTYPE) * lds * numSplitArraysMax * numSplitArraysMax; } __host__ void cuozblasMallocSplitMat ( cuozblasHandle_t *oh, const int32_t m, const int32_t n, const uint32_t numSplitArraysMax, FP_CTYPE **devSplit, FP_CTYPE **devMax, int32_t &lds ) { lds = getPitchSize (m); devSplit[0] = (FP_CTYPE*)(oh->devWork + oh->memAddr); oh->memAddr += sizeof(FP_CTYPE) * lds * n * (numSplitArraysMax + 1); // +1 is for output (work) devMax[0] = (FP_CTYPE*)(oh->devWork + oh->memAddr); oh->memAddr += sizeof(FP_CTYPE) * lds; // for Max } __host__ void cuozblasMallocResultMat ( cuozblasHandle_t *oh, const int32_t m, const int32_t n, const uint32_t numSplitArraysMax, FP_CTYPE **devSplit, int32_t &lds ) { lds = getPitchSize (m); devSplit[0] = (FP_CTYPE*)(oh->devWork + oh->memAddr); oh->memAddr += sizeof(FP_CTYPE) * lds * n * numSplitArraysMax * numSplitArraysMax; } __host__ void cuozblasMallocSplitMatSparse ( cuozblasHandle_t *oh, const int32_t m, const int32_t nnz, const uint32_t numSplitArraysMax, FP_CTYPE **devSplit, FP_CTYPE **devMax, int32_t &lds ) { lds = getPitchSize (nnz); devSplit[0] = (FP_CTYPE*)(oh->devWork + oh->memAddr); oh->memAddr += sizeof(FP_CTYPE) * lds * (numSplitArraysMax + 1); // +1 is for output (work) devMax[0] = (FP_CTYPE*)(oh->devWork + oh->memAddr); oh->memAddr += sizeof(FP_CTYPE) * getPitchSize(m); } ozblas-1.2a17/src/cuda/cuozblas_common.h0000644000175200017520000001320413725177602020531 0ustar mukunokimukunoki// setting #define DefaultWorkSize 8e9 // 8GB #define NumSplitDefaultMax 20 //#define SPARSITY_THRESHOLD 1 // [%] #define SPLIT_N_NTX 32 #define SPLIT_N_NTY 16 #define SPLIT_T_NTX 512 #define SPLIT_T_NTY 1 #define SPLIT_VEC_NTX 512 #define SPLIT_VEC_NBX 512 #include #include #define __STDC_FORMAT_MACROS #include #include #include #include #include #include #include #include #include "../../include/cuozblas.h" #include "cuozblas_internal.h" #if defined (PREC_S) #define FP_DTYPE float #define FP_CTYPE float #define UNIT 24 #define OzCG cuozblasScg #define OzDOT cuozblasSdot #define OzAXPY cuozblasSaxpy #define OzGEMV cuozblasSgemv #define OzGEMM cuozblasSgemm #define OzNRM2 cuozblasSnrm2 #define OzCSRMV cuozblasScsrmv #define OzCSRMVSplitA cuozblasScsrmvSplitA #define RCuAXPY cublasSaxpy #define RCuDOT cublasSdot #define RCuGEMV cublasSgemv #define RCuGEMM cublasSgemm #define RCuNRM2 cublasSnrm2 #define RCuCSRMV cusparseScsrmv #define CuAXPY cublasSaxpy #define CuDOT cublasSdot #define CuGEMV cublasSgemv #define CuGEMM cublasSgemm #define CuNRM2 cublasSnrm2 #define CuCSRMV cusparseScsrmv #define CuGEMMBatched cublasSgemmBatched #define CuCSRMM cusparseScsrmm #define CuDENSE2CSR cusparseSdense2csr #define CuASUM cublasSasum #define CuSCAL cublasSscal #define CuIAMAX cublasIsamax #define CuNNZ cusparseSnnz #define MuMDOT mublasSmdot #define ADD __fadd_rn #define SUB __fsub_rn #define MUL __fmul_rn #define FMA __fmaf_rn #define PdevKernel cuozblasSaxpyPdevKernel #define PhstKernel cuozblasSaxpyPhstKernel #define cuozblasMallocSplitMat cuozblasSMallocSplitMat #define cuozblasMallocSplitMatSparse cuozblasSMallocSplitMatSparse #define cuozblasMallocResultMat cuozblasSMallocResultMat #define cuozblasMallocSplitVec cuozblasSMallocSplitVec #define cuozblasMallocResultVec cuozblasSMallocResultVec #define cuozblasSplitMat cuzoblasSSplitMat #define cuozblasSplitMatSparse cuozblasSSplitMatSparse #define cuozblasSplitShiftMat cuzoblasSSplitShiftMat #define cuozblasSplitShiftMatSparse cuozblasSSplitShiftMatSparse #define cuozblasSumMat cuozblasSSumMat #define cuozblasPrintVec cuozblasSPrintVec #elif defined (PREC_D) #define FP_DTYPE double #define FP_CTYPE double #define UNIT 53 #define OzCG cuozblasDcg #define OzDOT cuozblasDdot #define OzAXPY cuozblasDaxpy #define OzGEMV cuozblasDgemv #define OzGEMM cuozblasDgemm #define OzNRM2 cuozblasDnrm2 #define OzCSRMV cuozblasDcsrmv #define OzCSRMVSplitA cuozblasDcsrmvSplitA #define RCuAXPY cublasDaxpy #define RCuDOT cublasDdot #define RCuGEMV cublasDgemv #define RCuGEMM cublasDgemm #define RCuNRM2 cublasDnrm2 #define RCuCSRMV cusparseDcsrmv #define CuAXPY cublasDaxpy #define CuDOT cublasDdot #define CuGEMV cublasDgemv #define CuGEMM cublasDgemm #define CuNRM2 cublasDnrm2 #define CuCSRMV cusparseDcsrmv #define CuGEMMBatched cublasDgemmBatched #define CuCSRMM cusparseDcsrmm #define CuDENSE2CSR cusparseDdense2csr #define CuASUM cublasDasum #define CuSCAL cublasDscal #define CuIAMAX cublasIdamax #define CuNNZ cusparseDnnz #define MuMDOT mublasDmdot #define ADD __dadd_rn #define SUB __dsub_rn #define MUL __dmul_rn #define FMA __fma_rn #define PdevKernel cuozblasDaxpyPdevKernel #define PhstKernel cuozblasDaxpyPhstKernel #define cuozblasMallocSplitMat cuozblasDMallocSplitMat #define cuozblasMallocSplitMatSparse cuozblasDMallocSplitMatSparse #define cuozblasMallocResultMat cuozblasDMallocResultMat #define cuozblasMallocSplitVec cuozblasDMallocSplitVec #define cuozblasMallocResultVec cuozblasDMallocResultVec #define cuozblasSplitMat cuzoblasDSplitMat #define cuozblasSplitMatSparse cuozblasDSplitMatSparse #define cuozblasSplitShiftMat cuzoblasDSplitShiftMat #define cuozblasSplitShiftMatSparse cuozblasDSplitShiftMatSparse #define cuozblasSumMat cuozblasDSumMat #define cuozblasPrintVec cuozblasDPrintVec #elif defined (PREC_SD) #define FP_DTYPE float #define FP_CTYPE double #define UNIT 53 #define OzCG cuozblasSDcg #define OzDOT cuozblasSDdot #define OzAXPY cuozblasSDaxpy #define OzGEMV cuozblasSDgemv #define OzGEMM cuozblasSDgemm #define OzNRM2 cuozblasSDnrm2 #define OzCSRMV cuozblasSDcsrmv #define OzCSRMVSplitA cuozblasSDcsrmvSplitA #define RCuAXPY cublasSaxpy #define RCuDOT cublasSdot #define RCuGEMV cublasSgemv #define RCuGEMM cublasSgemm #define RCuNRM2 cublasSnrm2 #define RCuCSRMV cusparseScsrmv #define CuAXPY cublasDaxpy #define CuDOT cublasDdot #define CuGEMV cublasDgemv #define CuGEMM cublasDgemm #define CuNRM2 cublasDnrm2 #define CuCSRMV cusparseDcsrmv #define CuGEMMBatched cublasDgemmBatched #define CuCSRMM cusparseDcsrmm #define CuDENSE2CSR cusparseDdense2csr #define CuASUM cublasDasum #define CuSCAL cublasSscal // <- 's' #define CuIAMAX cublasIdamax #define CuNNZ cusparseDnnz #define MuMDOT mublasDmdot #define ADD __dadd_rn #define SUB __dsub_rn #define MUL __dmul_rn #define FMA __fma_rn #define PdevKernel cuozblasSDaxpyPdevKernel #define PhstKernel cuozblasSDaxpyPhstKernel #define cuozblasMallocSplitMat cuozblasSDMallocSplitMat #define cuozblasMallocSplitMatSparse cuozblasSDMallocSplitMatSparse #define cuozblasMallocResultMat cuozblasSDMallocResultMat #define cuozblasMallocSplitVec cuozblasSDMallocSplitVec #define cuozblasMallocResultVec cuozblasSDMallocResultVec #define cuozblasSplitMat cuzoblasSDSplitMat #define cuozblasSplitMatSparse cuozblasSDSplitMatSparse #define cuozblasSplitShiftMat cuzoblasSDSplitShiftMat #define cuozblasSplitShiftMatSparse cuozblasSDSplitShiftMatSparse #define cuozblasSumMat cuozblasSDSumMat #define cuozblasPrintVec cuozblasSDPrintVec #endif ozblas-1.2a17/src/cuda/cuozblas_split.cu0000644000175200017520000003243513725177602020563 0ustar mukunokimukunoki#include "cuozblas_common.h" /* major: 'r' or 'c' -----> r | V c */ __forceinline__ __device__ double NextPowTwo (const double p) { return __dsub_rn(scalbn (p, UNIT), __dmul_rn(__dsub_rn(scalbn (1., UNIT), 1), p)); //return scalbn (p, UNIT) - ((scalbn (1., UNIT) - 1) * p); } // ========================================= // SplitVec // Based on K.Ozaki et al., "Error-free transformations of matrix multiplication // by using fast routines of matrix multiplication and its applications", 2012 // ========================================= __global__ void cuozblasSplitVecSubKernel ( int32_t n, FP_CTYPE *devMax ) { const size_t iTx = threadIdx.x; const size_t nTx = blockDim.x; extern __shared__ FP_CTYPE shm[]; FP_CTYPE tmp, max = 0.; if (iTx < n) max = devMax[iTx]; for (size_t i = nTx + iTx; i < n; i += nTx) { tmp = devMax[i]; if (max < fabs(tmp)) max = fabs(tmp); } shm[iTx] = max; __syncthreads (); #pragma unroll for (size_t i = nTx/2; i > 0; i >>= 1) { if (iTx < i && shm[iTx] < shm[iTx+i]) shm[iTx] = shm[iTx+i]; __syncthreads (); } if (iTx == 0) devMax[0] = shm[0]; } __global__ void cuozblasSplitVecKernel ( const int32_t n, const int32_t rho, const FP_CTYPE * __restrict__ devInput, FP_CTYPE *devOutput, FP_CTYPE *devSplit, const FP_CTYPE *devMaxOld, FP_CTYPE *devMax ) { const size_t iTx = threadIdx.x; const size_t iBx = blockIdx.x; const size_t nTx = blockDim.x; const size_t addrx = iBx * nTx + iTx; FP_CTYPE sigma, max, input, split; extern __shared__ FP_CTYPE shm[]; // nTx elements max = devMaxOld[0]; sigma = scalbn (1., rho) * NextPowTwo (max); max = 0.; for (size_t i = addrx; i < n; i += nTx * gridDim.x) { input = devInput[i]; split = SUB (ADD (input, sigma), sigma); input = SUB (input, split); devSplit[i] = split; devOutput[i] = input; if (max < fabs(input)) max = fabs(input); } shm[iTx] = max; __syncthreads (); #pragma unroll for (size_t i = nTx/2; i > 0; i>>=1) { if (iTx < i && shm[iTx] < shm[iTx+i]) shm[iTx] = shm[iTx+i]; __syncthreads (); } if (iTx == 0) devMax[iBx] = shm[0]; } __global__ void cuozblasSplitVecEarlyExitKernel ( const int32_t n, const int32_t rho, const FP_CTYPE * __restrict__ devInput, FP_CTYPE *devSplit, FP_CTYPE *devMax ) { const size_t iBx = blockIdx.x; const size_t nTx = blockDim.x; const size_t iTx = threadIdx.x; const size_t addrx = iBx * nTx + iTx; FP_CTYPE sigma, max, input, split; if (addrx < n) { max = devMax[0]; sigma = scalbn (1., rho) * NextPowTwo (max); input = devInput[addrx]; split = SUB (ADD (input, sigma), sigma); input = SUB (input, split); devSplit[addrx] = split; } } // ========================================= // SplitMat // ========================================= __global__ void cuozblasFindMaxMatNKernel ( const int32_t m, const int32_t n, const FP_CTYPE * __restrict__ devInput, const int32_t ldi, FP_CTYPE *devMax ) { const size_t iBx = blockIdx.x; const size_t nTx = blockDim.x; const size_t nTy = blockDim.y; const size_t iTx = threadIdx.x; const size_t iTy = threadIdx.y; const size_t addrx = iBx * nTx + iTx; FP_CTYPE max, input, tmp; extern __shared__ FP_CTYPE shm[]; size_t j; if (addrx < m){ max = 0.; for (j = iTy; j < n; j+=nTy) { input = devInput[j * ldi + addrx]; if (max < fabs(input)) max = fabs(input); } shm[nTx * iTy + iTx] = max; __syncthreads (); if (iTy == 0) { max = shm[iTx]; #pragma unroll for (j = 1; j < SPLIT_N_NTY; j++) { tmp = shm[nTx * j + iTx]; if (max < fabs(tmp)) max = fabs(tmp); } devMax[addrx] = max; } } } __global__ void cuozblasFindMaxMatTKernel ( const int32_t m, const int32_t n, const FP_CTYPE * __restrict__ devInput, const int32_t ldi, FP_CTYPE *devMax ) { const int32_t iBx = blockIdx.x; const int32_t iBy = blockIdx.y; const int32_t nTx = blockDim.x; const int32_t nTy = blockDim.y; const int32_t iTx = threadIdx.x; const int32_t iTy = threadIdx.y; const int32_t addrx = iBx * nTx + iTx; const int32_t addry = iBy * nTy + iTy; FP_CTYPE max, input, tmp; extern __shared__ FP_CTYPE shm[]; if (addry < n){ max = 0.; for (size_t i = addrx; i < m; i += nTx) { input = devInput[addry * ldi + i]; if (max < fabs(input)) max = fabs(input); } shm[iTx] = max; __syncthreads (); #pragma unroll for (size_t i = nTx/2; i > 0; i>>=1) { tmp = shm[iTx+i]; if (iTx < i && shm[iTx] < tmp) shm[iTx] = tmp; __syncthreads (); } if (iTx == 0) devMax[addry] = shm[0]; } } __host__ void cuozblasFindMaxMatDevice ( const char major, const int32_t m, const int32_t n, const FP_CTYPE *devInput, const int32_t ldi, FP_CTYPE *devMax ) { size_t ntx, nty, nbx, nby; dim3 threads, grid; if (major == 'r') { ntx = SPLIT_N_NTX; nty = SPLIT_N_NTY; nbx = ceil (float(m) / ntx); nby = 1; threads = dim3 (ntx, nty); grid = dim3 (nbx, nby); cuozblasFindMaxMatNKernel <<< grid, threads, sizeof(FP_CTYPE) * ntx * nty >>> (m, n, devInput, ldi, devMax); } else { ntx = SPLIT_T_NTX; nty = SPLIT_T_NTY; nbx = 1; nby = ceil (float(n) / nty); threads = dim3 (ntx, nty); grid = dim3 (nbx, nby); cuozblasFindMaxMatTKernel <<< grid, threads, sizeof(FP_CTYPE) * ntx >>> (m, n, devInput, ldi, devMax); } } __global__ void cuozblasSplitMatNKernel ( const int32_t m, const int32_t n, const int32_t rho, const FP_CTYPE * __restrict__ devInput, const int32_t ldi, FP_CTYPE *devOutput, FP_CTYPE *devSplit, const int32_t lds, FP_CTYPE *devMax ) { const size_t iBx = blockIdx.x; const size_t nTx = SPLIT_N_NTX; const size_t nTy = SPLIT_N_NTY; const size_t iTx = threadIdx.x; const size_t iTy = threadIdx.y; const size_t addrx = iBx * nTx + iTx; FP_CTYPE sigma, max, input, split, tmp; extern __shared__ FP_CTYPE shm[]; if (addrx < m){ // split ------------------------------------ max = devMax[addrx]; sigma = scalbn (1., rho) * NextPowTwo (max); max = 0.; for (size_t j = iTy; j < n; j+=nTy) { input = devInput[j * ldi + addrx]; split = SUB (ADD (input, sigma), sigma); input = SUB (input, split); devSplit[j * lds + addrx] = split; devOutput[j * lds + addrx] = input; if (max < fabs(input)) max = fabs(input); } shm[nTx * iTy + iTx] = max; __syncthreads (); if (iTy == 0) { max = shm[iTx]; #pragma unroll for (size_t j = 1; j < nTy; j++) { tmp = shm[nTx * j + iTx]; if (max < fabs(tmp)) max = fabs(tmp); } devMax[addrx] = max; } } } __global__ void cuozblasSplitMatTKernel ( const int32_t m, const int32_t n, const int32_t rho, const FP_CTYPE * __restrict__ devInput, const int32_t ldi, FP_CTYPE *devOutput, FP_CTYPE *devSplit, const int32_t lds, FP_CTYPE *devMax ) { const size_t iBx = blockIdx.x; const size_t iBy = blockIdx.y; const size_t nTx = blockDim.x; const size_t nTy = blockDim.y; const size_t iTx = threadIdx.x; const size_t iTy = threadIdx.y; const size_t addrx = iBx * nTx + iTx; const size_t addry = iBy * nTy + iTy; FP_CTYPE sigma, max, input, split; extern __shared__ FP_CTYPE shm[]; if (addry < n){ max = devMax[addry]; sigma = scalbn (1., rho) * NextPowTwo (max); max = 0.; for (size_t i = addrx; i < m; i += nTx) { input = devInput[addry * ldi + i]; split = SUB (ADD (input, sigma), sigma); input = SUB (input, split); devSplit[addry * lds + i] = split; devOutput[addry * lds + i] = input; if (max < fabs(input)) max = fabs(input); } shm[iTx] = max; __syncthreads (); #pragma unroll for (size_t i = nTx/2; i > 0; i>>=1) { if (iTx < i && shm[iTx] < shm[iTx+i]) shm[iTx] = shm[iTx+i]; __syncthreads (); } if (iTx == 0) devMax[addry] = shm[0]; } } __global__ void cuozblasSplitMatEarlyExitNKernel ( const int32_t m, const int32_t n, const int32_t rho, const FP_CTYPE * __restrict__ devInput, const int32_t ldi, FP_CTYPE *devSplit, const int32_t lds, FP_CTYPE *devMax ) { const size_t iBx = blockIdx.x; const size_t nTx = SPLIT_N_NTX; const size_t nTy = SPLIT_N_NTY; const size_t iTx = threadIdx.x; const size_t iTy = threadIdx.y; const size_t addrx = iBx * nTx + iTx; FP_CTYPE sigma, max, input, split; if (addrx < m){ max = devMax[addrx]; sigma = scalbn (1., rho) * NextPowTwo (max); for (size_t j = iTy; j < n; j+=nTy) { input = devInput[j * ldi + addrx]; split = SUB (ADD (input, sigma), sigma); input = SUB (input, split); devSplit[j * lds + addrx] = split; } } } __global__ void cuozblasSplitMatEarlyExitTKernel ( const int32_t m, const int32_t n, const int32_t rho, const FP_CTYPE * __restrict__ devInput, const int32_t ldi, FP_CTYPE *devSplit, const int32_t lds, FP_CTYPE *devMax ) { const size_t iBx = blockIdx.x; const size_t iBy = blockIdx.y; const size_t nTx = blockDim.x; const size_t nTy = blockDim.y; const size_t iTx = threadIdx.x; const size_t iTy = threadIdx.y; const size_t addrx = iBx * nTx + iTx; const size_t addry = iBy * nTy + iTy; FP_CTYPE sigma, max, input, split; if (addry < n){ max = devMax[addry]; sigma = scalbn (1., rho) * NextPowTwo (max); for (size_t i = addrx; i < m; i += nTx) { input = devInput[addry * ldi + i]; split = SUB (ADD (input, sigma), sigma); input = SUB (input, split); devSplit[addry * lds + i] = split; } } } __host__ void cuozblasSplitMatDevice ( const char major, const int32_t m, const int32_t n, const FP_CTYPE *devInput, // input matrix (devAwrk) const int32_t ldi, // leading dimension of input matrix FP_CTYPE *devOutput, // output matrix (devAwrk) FP_CTYPE *devSplit, // split matrices (output): this includes NumSplitMax matrices const int32_t lds, // leading dimension of split matrix (# of cols) FP_CTYPE *devMax, uint32_t lastFlag ) { size_t ntx, nty, nbx, nby; dim3 threads, grid; const int32_t dim = ((major == 'r') && n > 1) ? n : m; int32_t rho = ceil((double)(UNIT+log2((double)dim))/2.); if (major == 'r') { if (n == 1) { ntx = SPLIT_VEC_NTX; threads = dim3 (ntx); if (!lastFlag) { nbx = min ((int)ceil((float)m/ntx), SPLIT_VEC_NBX); // 512 grid = dim3 (nbx); FP_CTYPE *devMaxOld = devMax; // because devMax may be overwritten by other thread-blocks devMaxOld = &devMax[getPitchSize(SPLIT_VEC_NBX)]; cudaMemcpy (devMaxOld, devMax, sizeof(FP_CTYPE), cudaMemcpyDeviceToDevice); cuozblasSplitVecKernel <<< grid, threads, sizeof(FP_CTYPE) * ntx >>> (m, rho, devInput, devOutput, devSplit, devMaxOld, devMax); grid = dim3 (1); cuozblasSplitVecSubKernel <<< grid, threads, sizeof(FP_CTYPE) * ntx >>> (nbx, devMax); } else { nbx = ceil((float)m/ntx); grid = dim3 (nbx); cuozblasSplitVecEarlyExitKernel <<< grid, threads >>> (m, rho, devInput, devSplit, devMax); } } else { // printf ("\ndevMaxA"); // PrintMat (n, 1, devMax, n); ntx = SPLIT_N_NTX; nty = SPLIT_N_NTY; nbx = ceil (float(m) / ntx); nby = 1; threads = dim3 (ntx, nty); grid = dim3 (nbx, nby); if (!lastFlag) cuozblasSplitMatNKernel <<< grid, threads, sizeof(FP_CTYPE) * ntx * nty >>> (m, n, rho, devInput, ldi, devOutput, devSplit, lds, devMax); else cuozblasSplitMatEarlyExitNKernel <<< grid, threads >>> (m, n, rho, devInput, ldi, devSplit, lds, devMax); } } else { ntx = SPLIT_T_NTX; nty = SPLIT_T_NTY; nbx = 1; nby = ceil (float(n) / nty); threads = dim3 (ntx, nty); grid = dim3 (nbx, nby); if (!lastFlag) cuozblasSplitMatTKernel <<< grid, threads, sizeof(FP_CTYPE) * ntx * 2 >>> (m, n, rho, devInput, ldi, devOutput, devSplit, lds, devMax); else cuozblasSplitMatEarlyExitTKernel <<< grid, threads >>> (m, n, rho, devInput, ldi, devSplit, lds, devMax); } } __host__ void cuozblasSplitMat ( cuozblasHandle_t *oh, const char major, const int32_t m, const int32_t n, const FP_CTYPE *devInput, const int32_t ld, FP_CTYPE *devSplit, FP_CTYPE *devMax, const int32_t lds, uint32_t &numSplitArrays, uint32_t &lastFlag // note: numSplitArrays and lastFlag cannot be included in oh because this function is used for both row-split and col-split /* work: devSplit[0] input: devSplit[0] split: devSplit[i] */ ) { uint32_t numSplitArraysMax = numSplitArrays; uint32_t i = 1; FP_CTYPE check; // FindMax^(0) if (n == 1 && major == 'r') { int32_t ptrMax = 0; CuIAMAX (oh->ch, m, devInput, 1, &ptrMax); // Warning: note that cublas returns the index on the basis of 1-origin, while MKL (CBLAS) returns it on the basis of 0-origin !! cudaMemcpy (devMax, &devInput[ptrMax-1], sizeof(FP_CTYPE), cudaMemcpyDeviceToDevice); // so here do -1 !! } else { cuozblasFindMaxMatDevice (major, m, n, devInput, ld, devMax); } // Split^(0) & FindMax^(1) lastFlag = (oh->reproModeFlag && numSplitArraysMax == 1) ? 1 : 0; cuozblasSplitMatDevice (major, m, n, devInput, ld, &devSplit[lds*n], &devSplit[0], lds, devMax, lastFlag); while (i < numSplitArraysMax) { // Check if (numSplitArraysMax-1 == i && !oh->reproModeFlag) { i++; break;} // when repromode=0 if (n == 1 && major == 'r') cudaMemcpy (&check, devMax, sizeof(FP_CTYPE), cudaMemcpyDeviceToHost); else CuASUM (oh->ch, (major=='r')?m:n, devMax, 1, &check); if (fabs(check) == 0) break; // Split^(i) & FindMax^(i+1) lastFlag = (oh->reproModeFlag && numSplitArraysMax-1 == i) ? 1 : 0; cuozblasSplitMatDevice (major, m, n, &devSplit[lds*n*i], lds, &devSplit[lds*n*(i+1)], &devSplit[lds*n*i], lds, devMax, lastFlag); i++; } numSplitArrays = i; if (lastFlag == 1 && oh->numSplitArraysMax == 0) { fprintf (stderr, "* split error: out of memory (the result may be not correct rounding)\n"); exit (1); } } #include "cuozblas_split_shift.cu" ozblas-1.2a17/src/cuda/Makefile0000644000175200017520000000420613725177602016630 0ustar mukunokimukunoki# Makefile include ../../make.inc NVCC = $(CUDA_PATH)/bin/nvcc AR = ar LIB := ../../lib/libcuozblas.a FLAGS := -O3 NVCCFLAGS := -O3 -DCUDA all: blas ifeq ($(CC30),yes) NVCCFLAGS += -gencode arch=compute_30,code=sm_30 endif ifeq ($(CC32),yes) NVCCFLAGS += -gencode arch=compute_32,code=sm_32 endif ifeq ($(CC35),yes) NVCCFLAGS += -gencode arch=compute_35,code=sm_35 endif ifeq ($(CC37),yes) NVCCFLAGS += -gencode arch=compute_37,code=sm_37 endif ifeq ($(CC50),yes) NVCCFLAGS += -gencode arch=compute_50,code=sm_50 endif ifeq ($(CC52),yes) NVCCFLAGS += -gencode arch=compute_52,code=sm_52 endif ifeq ($(CC53),yes) NVCCFLAGS += -gencode arch=compute_53,code=sm_53 endif ifeq ($(CC60),yes) NVCCFLAGS += -gencode arch=compute_60,code=sm_60 endif ifeq ($(CC61),yes) NVCCFLAGS += -gencode arch=compute_61,code=sm_61 endif ifeq ($(CC70),yes) NVCCFLAGS += -gencode arch=compute_60,code=sm_70 endif ifeq ($(CC75),yes) NVCCFLAGS += -gencode arch=compute_60,code=sm_75 endif ALLSRC = \ cuozblas_auxiliary.cu \ cuozblas_split.cu \ cuozblas_sum.cu ALLHDR = \ cuozblas_common.h \ cuozblas_split_shift.cu \ cuozblas_internal.h ifeq ($(AXPY),yes) ALLSRC += cuozblas_axpy.cu endif ifeq ($(DOT),yes) ALLSRC += cuozblas_dot.cu ALLHDR += blas/mublas_dot.cu ifeq ($(NRM2),yes) ALLSRC += cuozblas_nrm2.cu endif endif ifeq ($(GEMV),yes) ALLSRC += cuozblas_gemv.cu endif ifeq ($(GEMM),yes) ALLSRC += cuozblas_gemm.cu endif ifeq ($(CSRMV),yes) ALLSRC += cuozblas_split_sparse.cu \ cuozblas_csrmv.cu ALLHDR += cuozblas_split_sparse_shift.cu ifeq ($(CG),yes) ALLSRC += cuozblas_cg.cu endif endif ALLOBJ = cuozblas.o ifeq ($(PREC_S),yes) ALLOBJ += $(ALLSRC:.cu=.s.o) endif ifeq ($(PREC_D),yes) ALLOBJ += $(ALLSRC:.cu=.d.o) endif ifeq ($(PREC_SD),yes) ALLOBJ += $(ALLSRC:.cu=.sd.o) endif $(ALLOBJ) : $(ALLHDR) blas: $(ALLOBJ) $(AR) rv $(LIB) $(ALLOBJ) #%.o: %.cu # $(NVCC) $(NVCCFLAGS) -o $@ -c $< %.s.o: %.cu $(NVCC) $(NVCCFLAGS) -o $@ -c $< -DPREC_S %.d.o: %.cu $(NVCC) $(NVCCFLAGS) -o $@ -c $< -DPREC_D %.sd.o: %.cu $(NVCC) $(NVCCFLAGS) -o $@ -c $< -DPREC_SD cuozblas.o: cuozblas.cpp $(CXX) $(FLAGS) -o $@ -c $< -I$(CUDA_PATH)/include clean : $(RM) *.o $(RM) $(LIB) ozblas-1.2a17/src/cuda/cuozblas.cpp0000644000175200017520000000657213725177602017526 0ustar mukunokimukunoki#include "cuozblas_common.h" double timer () { struct timeval tv; cudaDeviceSynchronize (); gettimeofday (&tv, NULL); return tv.tv_sec + (double) tv.tv_usec * 1.0e-6; } size_t getPitchSize (size_t n) { return ceil((double)n / 128) * 128; } uint32_t memCheck (cuozblasHandle_t *oh) { if (oh->memAddr > oh->workSizeBytes) return 1; return 0; } void counterInit (cuozblasHandle_t *oh) { oh->t_SplitA = 0.; oh->t_SplitB = 0.; oh->t_comp = 0.; oh->t_sum = 0.; oh->t_total = 0.; oh->numSplitArraysA = 0; oh->numSplitArraysB = 0; oh->numSplitArraysC = 0; oh->lastFlagA = 0; oh->lastFlagB = 0; oh->memAddr = oh->memMaskSplitA; } void cuozblasCreate (cuozblasHandle_t *oh, size_t WorkSizeBytes, uint32_t NumSplitArraysMax, uint32_t FastMode, uint32_t ReproMode, uint32_t SumMode) { oh->workSizeBytes = (WorkSizeBytes) ? WorkSizeBytes : DefaultWorkSize; oh->numSplitArraysMax = NumSplitArraysMax; oh->fastModeFlag = FastMode; oh->reproModeFlag = ReproMode; oh->sumModeFlag = SumMode; // for CG oh->memMaskSplitA = 0; // disable pre-split of matA oh->splitShift = 1; // default (no-Splitshift) // experimental parameters (default) oh->useGemmFlag = 1; oh->useMyGemmFlag = 1; oh->useBatchedGemmFlag = 1; oh->sparseThreshold = 0.; // work memory allocation cudaMalloc ((void **) &oh->devWorkCommon, sizeof(double)*16);// for NRM2 cudaMalloc ((void **) &oh->devWork, oh->workSizeBytes); // cuBLAS cublasCreate (&oh->ch); cublasSetPointerMode(oh->ch, CUBLAS_POINTER_MODE_HOST); cusparseCreate (&oh->csh); cusparseSetPointerMode(oh->csh, CUSPARSE_POINTER_MODE_HOST); // batched BLAS if (oh->useBatchedGemmFlag) { char *hstBatchAddr_; cudaMallocHost ((void **) &hstBatchAddr_, sizeof(double*) * NumSplitDefaultMax * NumSplitDefaultMax * 3); oh->hstBatchAddr = hstBatchAddr_; char *devBatchAddr_; cudaMalloc ((void **) &devBatchAddr_, sizeof(double*) * NumSplitDefaultMax * NumSplitDefaultMax * 3); oh->devBatchAddr = devBatchAddr_; } } void cuozblasProfilingSetting (cuozblasHandle_t *oh, uint32_t useGemmFlag, uint32_t useMyGemmFlag, uint32_t useBatchedGemmFlag) { oh->useGemmFlag = useGemmFlag; oh->useMyGemmFlag = useMyGemmFlag; oh->useBatchedGemmFlag = useBatchedGemmFlag; } void cuozblasProfilingSetting2 (cuozblasHandle_t *oh, float sparseThreshold) { oh->sparseThreshold = sparseThreshold; } void cuozblasSetNumSplitArraysMax (cuozblasHandle_t *oh, uint32_t NumSplitArraysMax) { oh->numSplitArraysMax = NumSplitArraysMax; } void cuozblasSetFastMode (cuozblasHandle_t *oh, uint32_t FastMode) { oh->fastModeFlag = (FastMode == 0) ? 0 : 1; } void cuozblasSetReproMode (cuozblasHandle_t *oh, uint32_t ReproMode) { oh->reproModeFlag = (ReproMode == 0) ? 0 : 1; } void cuozblasSetSumMode (cuozblasHandle_t *oh, uint32_t SumModeFlag) { oh->sumModeFlag = (SumModeFlag == 0) ? 0 : 1; } uint32_t cuozblasGetNumSplitArraysMax (cuozblasHandle_t *oh) { return oh->numSplitArraysMax; } uint32_t cuozblasGetFastMode (cuozblasHandle_t *oh) { return oh->fastModeFlag; } uint32_t cuozblasGetReproMode (cuozblasHandle_t *oh) { return oh->reproModeFlag; } uint32_t cuozblasGetSumMode (cuozblasHandle_t *oh) { return oh->sumModeFlag; } void cuozblasDestroy (cuozblasHandle_t *oh) { cudaFree (oh->devWork); cudaFree (oh->devWorkCommon); if (oh->useBatchedGemmFlag) { cudaFreeHost (oh->hstBatchAddr); cudaFree (oh->devBatchAddr); } cublasDestroy (oh->ch); cusparseDestroy (oh->csh); } ozblas-1.2a17/src/cuda/cuozblas_sum.cu0000644000175200017520000001543313725177602020233 0ustar mukunokimukunoki#include "cuozblas_common.h" // ========================================= // NearSum (and AccSum) // from "ACCURATE FLOATING-POINT SUMMATION" by S.M.RUMP, T.OGITA, S.OISHI (2005) // http://www.ti3.tu-harburg.de/paper/rump/RuOgOi06.pdf // ========================================= __device__ __forceinline__ void cuTransform (const int32_t n, FP_CTYPE *vec, const int32_t ld, double rho, double &tau1, double &tau2) { int32_t i, m; double tmp, mu, sigma, t, tau; mu = fabs(vec[0]); for (i = 1; i < n; i++) mu = max (mu, fabs(vec[i*ld])); if ((n == 0) || (mu == 0.)) { tau1 = rho; tau2 = 0.; return; } m = ceil (log2((double)(n+2))); sigma = scalbn (1., m+ceil(log2(mu))); t = rho; while (1) { // ExtractVector tau = 0.; for (i = 0; i < n; i++) { tmp = (sigma + vec[i*ld]) - sigma; vec[i*ld] -= tmp; // <- output tau += tmp; } // here, tau1 = t1 tau1 = t + tau; if ((sigma <= scalbn (1., -1022)) || (fabs(tau1) >= scalbn (1., 2*m+1-53)*sigma)) { //FastTwoSum (t, tau, &tau1, &tau2); //tau1 = t + tau tau2 = tau - (tau1 - t); return; } sigma = scalbn (1., m-53) * sigma; t = tau1; } } __device__ __forceinline__ void cuTransformK (const int32_t n, FP_CTYPE *vec, const int32_t ld, double rho, double &res, double &r) { int32_t i; double tmp, tau1, tau2; cuTransform (n, vec, ld, rho, tau1, tau2); tmp = 0.; for (i = 0; i < n; i++) tmp += vec[i*ld]; res = tau1 + (tau2 + tmp); r = tau2 - (res - tau1); } __device__ __forceinline__ double cuSign (double v) { return (v < 0) ? -1.:1.; } __device__ double cuNearSum (const int32_t n, FP_CTYPE *vec, const int32_t ld) { double tmp, res, res2, r, r2, mu, delta, delta2; double eps = scalbn (1., -53); cuTransformK (n, vec, ld, 0, res, r); cuTransformK (n, vec, ld, r, delta, r2); if (delta == 0) return res; res2 = res + cuSign (delta) * eps * fabs(res); if (res2 == res) { mu = cuSign (delta) * eps * fabs(res); res2 = res + 2. * cuSign (delta) * eps * fabs(res); } else { mu = (res2 - res) / 2.; } if (fabs(delta) < fabs(mu)) return res; if (fabs(delta) > fabs(mu)) return res2; cuTransformK (n, vec, ld, r2, delta2, tmp); if (delta2 == 0) return res + mu; if (cuSign (delta2) == cuSign (mu)) return res2; return res; } // ========================================= // NearSumMat // ========================================= __global__ void cuNearSumMatPhstKernel ( const int32_t m, const int32_t n, FP_CTYPE *devCsplit, const int32_t llsc, const int32_t ldsc, const int32_t numsplit, FP_DTYPE *devC, const int32_t ldc, const FP_DTYPE alpha, const FP_DTYPE beta ) { const size_t iBx = blockIdx.x; const size_t iBy = blockIdx.y; const size_t nTx = blockDim.x; const size_t nTy = blockDim.y; const size_t iTx = threadIdx.x; const size_t iTy = threadIdx.y; const size_t addrx = iBx * nTx + iTx; const size_t addry = iBy * nTy + iTy; FP_CTYPE t; if (addrx < m && addry < n) { t = cuNearSum (numsplit, &devCsplit[addry * ldsc + addrx], llsc); if (beta == 0.) devC[addry * ldc + addrx] = MUL ((FP_CTYPE)alpha, t); else devC[addry * ldc + addrx] = FMA ((FP_CTYPE)alpha, t, MUL ((FP_CTYPE)beta, (FP_CTYPE)devC[addry * ldc + addrx])); } } __global__ void cuNearSumMatPdevKernel ( const int32_t m, const int32_t n, FP_CTYPE *devCsplit, const int32_t llsc, const int32_t ldsc, const int32_t numsplit, FP_DTYPE *devC, const int32_t ldc, const FP_DTYPE *alpha, const FP_DTYPE *beta ) { const size_t iBx = blockIdx.x; const size_t iBy = blockIdx.y; const size_t nTx = blockDim.x; const size_t nTy = blockDim.y; const size_t iTx = threadIdx.x; const size_t iTy = threadIdx.y; const size_t addrx = iBx * nTx + iTx; const size_t addry = iBy * nTy + iTy; FP_CTYPE t; if (addrx < m && addry < n) { t = cuNearSum (numsplit, &devCsplit[addry * ldsc + addrx], llsc); if (beta[0] == 0.) devC[addry * ldc + addrx] = MUL ((FP_CTYPE)alpha[0], t); else devC[addry * ldc + addrx] = FMA ((FP_CTYPE)alpha[0], t, MUL ((FP_CTYPE)beta[0], (FP_CTYPE)devC[addry * ldc + addrx])); } } __global__ void cuSumMatPhstKernel ( const int32_t m, const int32_t n, const FP_CTYPE * __restrict__ devCsplit, const int32_t llsc, const int32_t ldsc, const int32_t numsplit, FP_DTYPE *devC, const int32_t ldc, const FP_DTYPE alpha, const FP_DTYPE beta ) { const size_t iBx = blockIdx.x; const size_t iBy = blockIdx.y; const size_t nTx = blockDim.x; const size_t nTy = blockDim.y; const size_t iTx = threadIdx.x; const size_t iTy = threadIdx.y; const size_t addrx = iBx * nTx + iTx; const size_t addry = iBy * nTy + iTy; FP_CTYPE t = 0.; if (addrx < m && addry < n) { for (int32_t i = numsplit-1; i >= 0; i--) t += (FP_CTYPE)devCsplit[llsc * i + addry * ldsc + addrx]; if (beta == 0.) devC[addry * ldc + addrx] = MUL ((FP_CTYPE)alpha, t); else devC[addry * ldc + addrx] = FMA ((FP_CTYPE)alpha, t, MUL ((FP_CTYPE)beta, (FP_CTYPE)devC[addry * ldc + addrx])); } } __global__ void cuSumMatPdevKernel ( const int32_t m, const int32_t n, const FP_CTYPE * __restrict__ devCsplit, const int32_t llsc, const int32_t ldsc, const int32_t numsplit, FP_DTYPE *devC, const int32_t ldc, const FP_DTYPE *alpha, const FP_DTYPE *beta ) { const size_t iBx = blockIdx.x; const size_t iBy = blockIdx.y; const size_t nTx = blockDim.x; const size_t nTy = blockDim.y; const size_t iTx = threadIdx.x; const size_t iTy = threadIdx.y; const size_t addrx = iBx * nTx + iTx; const size_t addry = iBy * nTy + iTy; FP_CTYPE t = 0.; if (addrx < m && addry < n) { for (int32_t i = numsplit-1; i >= 0; i--) t += (FP_CTYPE)devCsplit[llsc * i + addry * ldsc + addrx]; if (beta[0] == 0.) devC[addry * ldc + addrx] = MUL ((FP_CTYPE)alpha[0], t); else devC[addry * ldc + addrx] = FMA ((FP_CTYPE)alpha[0], t, MUL ((FP_CTYPE)beta[0], (FP_CTYPE)devC[addry * ldc + addrx])); } } __host__ void cuozblasSumMat ( const int32_t m, const int32_t n, FP_CTYPE *devCsplit, const int32_t llsc, const int32_t ldsc, const int32_t numsplit, FP_DTYPE *devC, const int32_t ldc, const FP_DTYPE *alpha, const FP_DTYPE *beta, cublasPointerMode_t pmode, const uint32_t sumModeFlag ) { dim3 threads, grid; size_t ntx, nty, nbx, nby; ntx = 512; nty = 1; nbx = ceil (float(m) / ntx); nby = ceil (float(n) / nty); threads = dim3 (ntx, nty); grid = dim3 (nbx, nby); if (pmode == CUBLAS_POINTER_MODE_DEVICE) { if (sumModeFlag) cuNearSumMatPdevKernel <<< grid, threads >>> (m, n, devCsplit, llsc, ldsc, numsplit, devC, ldc, alpha, beta); else cuSumMatPdevKernel <<< grid, threads >>> (m, n, devCsplit, llsc, ldsc, numsplit, devC, ldc, alpha, beta); } else { if (sumModeFlag) cuNearSumMatPhstKernel <<< grid, threads >>> (m, n, devCsplit, llsc, ldsc, numsplit, devC, ldc, *alpha, *beta); else cuSumMatPhstKernel <<< grid, threads >>> (m, n, devCsplit, llsc, ldsc, numsplit, devC, ldc, *alpha, *beta); } } ozblas-1.2a17/src/cuda/cuozblas_internal.h0000644000175200017520000001614213725177602021061 0ustar mukunokimukunoki#ifndef CUOZBLAS_INTERNAL_H #define CUOZBLAS_INTERNAL_H double timer (); size_t getPitchSize (size_t n); void counterInit (cuozblasHandle_t *oh); uint32_t memCheck (cuozblasHandle_t *oh); cublasOperation_t ToCublasOp (char tran); char FromCublasOp (cublasOperation_t tran); #if defined (PREC_D) void cuozblasDMallocSplitMat ( cuozblasHandle_t *oh, const int32_t m, const int32_t n, const uint32_t numSplitArraysMax, double **devSplit, double **devMax, int32_t &lds); void cuozblasDMallocSplitMatSparse ( cuozblasHandle_t *oh, const int32_t m, const int32_t nnz, const uint32_t numSplitArraysMax, double **devSplit, double **devMax, int32_t &lds); void cuozblasDMallocResultMat ( cuozblasHandle_t *oh, const int32_t m, const int32_t n, const uint32_t numSplitArraysMax, double **devSplit, int32_t &lds); void cuozblasDMallocSplitVec ( cuozblasHandle_t *oh, const int32_t n, const uint32_t numSplitArraysMax, double **devSplit, double **devMax, int32_t &lds); void cuozblasDMallocResultVec ( cuozblasHandle_t *oh, const int32_t n, const uint32_t numSplitArraysMax, double **devSplit, int32_t &lds); void cuzoblasDSplitMat ( cuozblasHandle_t *oh, const char major, const int32_t m, const int32_t n, const double *devInput, const int32_t ld, double *devSplit, double *devMax, const int32_t lds, uint32_t &numSplitArrays, uint32_t &lastFlag); void cuozblasDSplitMatSparse ( cuozblasHandle_t *oh, const char major, const int32_t m, const double *devInput, const int32_t *devRowptr, double *devSplit, double *devMax, const int32_t lds, uint32_t &numSplitArrays, uint32_t &lastFlag); void cuzoblasDSplitShiftMat ( cuozblasHandle_t *oh, const char major, const int32_t m, const int32_t n, const double *devInput, const int32_t ld, double *devSplit, double *devMax, const int32_t lds, uint32_t &numSplitArrays, uint32_t &lastFlag); void cuozblasDSplitShiftMatSparse ( cuozblasHandle_t *oh, const char major, const int32_t m, const double *devInput, const int32_t *devRowptr, double *devSplit, double *devMax, const int32_t lds, uint32_t &numSplitArrays, uint32_t &lastFlag); void cuozblasDSumMat ( const int32_t m, const int32_t n, double *devCsplit, const int32_t llsc, const int32_t ldsc, const int32_t numsplit, double *devC, const int32_t ldc, const double *alpha, const double *beta, const cublasPointerMode_t pmode, const uint32_t sumMode); void cuozblasDPrintVec ( const int32_t n, const double *devC); #elif defined (PREC_S) void cuozblasSMallocSplitMat ( cuozblasHandle_t *oh, const int32_t m, const int32_t n, const uint32_t numSplitArraysMax, float **devSplit, float **devMax, int32_t &lds); void cuozblasSMallocSplitMatSparse ( cuozblasHandle_t *oh, const int32_t m, const int32_t nnz, const uint32_t numSplitArraysMax, float **devSplit, float **devMax, int32_t &lds); void cuozblasSMallocResultMat ( cuozblasHandle_t *oh, const int32_t m, const int32_t n, const uint32_t numSplitArraysMax, float **devSplit, int32_t &lds); void cuozblasSMallocSplitVec ( cuozblasHandle_t *oh, const int32_t n, const uint32_t numSplitArraysMax, float **devSplit, float **devMax, int32_t &lds); void cuozblasSMallocResultVec ( cuozblasHandle_t *oh, const int32_t n, const uint32_t numSplitArraysMax, float **devSplit, int32_t &lds); void cuzoblasSSplitMat ( cuozblasHandle_t *oh, const char major, const int32_t m, const int32_t n, const float *devInput, const int32_t ld, float *devSplit, float *devMax, const int32_t lds, uint32_t &numSplitArrays, uint32_t &lastFlag); void cuozblasSSplitMatSparse ( cuozblasHandle_t *oh, const char major, const int32_t m, const float *devInput, const int32_t *devRowptr, float *devSplit, float *devMax, const int32_t lds, uint32_t &numSplitArrays, uint32_t &lastFlag); void cuzoblasSSplitShiftMat ( cuozblasHandle_t *oh, const char major, const int32_t m, const int32_t n, const float *devInput, const int32_t ld, float *devSplit, float *devMax, const int32_t lds, uint32_t &numSplitArrays, uint32_t &lastFlag); void cuozblasSSplitShiftMatSparse ( cuozblasHandle_t *oh, const char major, const int32_t m, const float *devInput, const int32_t *devRowptr, float *devSplit, float *devMax, const int32_t lds, uint32_t &numSplitArrays, uint32_t &lastFlag); void cuozblasSSumMat ( const int32_t m, const int32_t n, float *devCsplit, const int32_t llsc, const int32_t ldsc, const int32_t numsplit, float *devC, const int32_t ldc, const float *alpha, const float *beta, const cublasPointerMode_t pmode, const uint32_t sumMode); void cuozblasSPrintVec ( const int32_t n, const float *devC); #elif defined (PREC_SD) void cuozblasSDMallocSplitMat ( cuozblasHandle_t *oh, const int32_t m, const int32_t n, const uint32_t numSplitArraysMax, double **devSplit, double **devMax, int32_t &lds); void cuozblasSDMallocSplitMatSparse ( cuozblasHandle_t *oh, const int32_t m, const int32_t nnz, const uint32_t numSplitArraysMax, double **devSplit, double **devMax, int32_t &lds); void cuozblasSDMallocTmpMat ( cuozblasHandle_t *oh, const int32_t m, const int32_t n, double **devSplit, int32_t &lds); void cuozblasSDMallocTmpVec ( cuozblasHandle_t *oh, const int32_t n, double **devSplit, int32_t &lds); void cuozblasSDCopyMat ( const int32_t m, const int32_t n, const float *devA, const int32_t lda, double *devB, const int32_t ldb); void cuozblasSDMallocResultMat ( cuozblasHandle_t *oh, const int32_t m, const int32_t n, const uint32_t numSplitArraysMax, double **devSplit, int32_t &lds); void cuozblasSDMallocSplitVec ( cuozblasHandle_t *oh, const int32_t n, const uint32_t numSplitArraysMax, double **devSplit, double **devMax, int32_t &lds); void cuozblasSDMallocResultVec ( cuozblasHandle_t *oh, const int32_t n, const uint32_t numSplitArraysMax, double **devSplit, int32_t &lds); void cuzoblasSDSplitMat ( cuozblasHandle_t *oh, const char major, const int32_t m, const int32_t n, const double *devInput, const int32_t ld, double *devSplit, double *devMax, const int32_t lds, uint32_t &numSplitArrays, uint32_t &lastFlag); void cuozblasSDSplitMatSparse ( cuozblasHandle_t *oh, const char major, const int32_t m, const double *devInput, const int32_t *devRowptr, double *devSplit, double *devMax, const int32_t lds, uint32_t &numSplitArrays, uint32_t &lastFlag); void cuzoblasSDSplitShiftMat ( cuozblasHandle_t *oh, const char major, const int32_t m, const int32_t n, const double *devInput, const int32_t ld, double *devSplit, double *devMax, const int32_t lds, uint32_t &numSplitArrays, uint32_t &lastFlag); void cuozblasSDSplitShiftMatSparse ( cuozblasHandle_t *oh, const char major, const int32_t m, const double *devInput, const int32_t *devRowptr, double *devSplit, double *devMax, const int32_t lds, uint32_t &numSplitArrays, uint32_t &lastFlag); void cuozblasSDSumMat ( const int32_t m, const int32_t n, double *devCsplit, const int32_t llsc, const int32_t ldsc, const int32_t numsplit, float *devC, const int32_t ldc, const float *alpha, const float *beta, const cublasPointerMode_t pmode, const uint32_t sumMode); void cuozblasSDPrintVec ( const int32_t n, const float *devC); #endif //void cuozblasInitVecDevice ( const size_t n, int64_t *devA); void PrintMat ( const int32_t m, const int32_t n, const double *devC, const int32_t ldd); void PrintMatInt ( const int32_t m, const int32_t n, const int32_t *devC, const int32_t ldd); #endif ozblas-1.2a17/src/cuda/cuozblas_split_shift.cu0000644000175200017520000002465613725177602021766 0ustar mukunokimukunoki// ========================================= // SplitVec // Based on K.Ozaki et al., "Error-free transformations of matrix multiplication // by using fast routines of matrix multiplication and its applications", 2012 // ========================================= __global__ void cuozblasSplitShiftVecKernel ( const int32_t n, const int32_t rho, const FP_CTYPE * __restrict__ devInput, FP_CTYPE *devOutput, FP_CTYPE *devSplit, const FP_CTYPE *devMaxOld, FP_CTYPE *devMax, const int32_t splitShift ) { const size_t iTx = threadIdx.x; const size_t iBx = blockIdx.x; const size_t nTx = blockDim.x; const size_t addrx = iBx * nTx + iTx; FP_CTYPE sigma, max, input, split; extern __shared__ FP_CTYPE shm[]; // nTx elements max = devMaxOld[0]; sigma = scalbn (1., rho) * NextPowTwo (max) * splitShift; max = 0.; for (size_t i = addrx; i < n; i += nTx * gridDim.x) { input = devInput[i]; split = SUB (ADD (input, sigma), sigma); input = SUB (input, split); devSplit[i] = split; devOutput[i] = input; if (max < fabs(input)) max = fabs(input); } shm[iTx] = max; __syncthreads (); #pragma unroll for (size_t i = nTx/2; i > 0; i>>=1) { if (iTx < i && shm[iTx] < shm[iTx+i]) shm[iTx] = shm[iTx+i]; __syncthreads (); } if (iTx == 0) devMax[iBx] = shm[0]; } __global__ void cuozblasSplitShiftVecEarlyExitKernel ( const int32_t n, const int32_t rho, const FP_CTYPE * __restrict__ devInput, FP_CTYPE *devSplit, FP_CTYPE *devMax, const int32_t splitShift ) { const size_t iBx = blockIdx.x; const size_t nTx = blockDim.x; const size_t iTx = threadIdx.x; const size_t addrx = iBx * nTx + iTx; FP_CTYPE sigma, max, input, split; if (addrx < n) { max = devMax[0]; sigma = scalbn (1., rho) * NextPowTwo (max) * splitShift; input = devInput[addrx]; split = SUB (ADD (input, sigma), sigma); input = SUB (input, split); devSplit[addrx] = split; } } // ========================================= // SplitMat // ========================================= __global__ void cuozblasSplitShiftMatNKernel ( const int32_t m, const int32_t n, const int32_t rho, const FP_CTYPE * __restrict__ devInput, const int32_t ldi, FP_CTYPE *devOutput, FP_CTYPE *devSplit, const int32_t lds, FP_CTYPE *devMax, const int32_t splitShift ) { const size_t iBx = blockIdx.x; const size_t nTx = SPLIT_N_NTX; const size_t nTy = SPLIT_N_NTY; const size_t iTx = threadIdx.x; const size_t iTy = threadIdx.y; const size_t addrx = iBx * nTx + iTx; FP_CTYPE sigma, max, input, split, tmp; extern __shared__ FP_CTYPE shm[]; if (addrx < m){ // split ------------------------------------ max = devMax[addrx]; sigma = scalbn (1., rho) * NextPowTwo (max) * splitShift; max = 0.; for (size_t j = iTy; j < n; j+=nTy) { input = devInput[j * ldi + addrx]; split = SUB (ADD (input, sigma), sigma); input = SUB (input, split); devSplit[j * lds + addrx] = split; devOutput[j * lds + addrx] = input; if (max < fabs(input)) max = fabs(input); } shm[nTx * iTy + iTx] = max; __syncthreads (); if (iTy == 0) { max = shm[iTx]; #pragma unroll for (size_t j = 1; j < nTy; j++) { tmp = shm[nTx * j + iTx]; if (max < fabs(tmp)) max = fabs(tmp); } devMax[addrx] = max; } } } __global__ void cuozblasSplitShiftMatTKernel ( const int32_t m, const int32_t n, const int32_t rho, const FP_CTYPE * __restrict__ devInput, const int32_t ldi, FP_CTYPE *devOutput, FP_CTYPE *devSplit, const int32_t lds, FP_CTYPE *devMax, const int32_t splitShift ) { const size_t iBx = blockIdx.x; const size_t iBy = blockIdx.y; const size_t nTx = blockDim.x; const size_t nTy = blockDim.y; const size_t iTx = threadIdx.x; const size_t iTy = threadIdx.y; const size_t addrx = iBx * nTx + iTx; const size_t addry = iBy * nTy + iTy; FP_CTYPE sigma, max, input, split; extern __shared__ FP_CTYPE shm[]; if (addry < n){ max = devMax[addry]; sigma = scalbn (1., rho) * NextPowTwo (max) * splitShift; max = 0.; for (size_t i = addrx; i < m; i += nTx) { input = devInput[addry * ldi + i]; split = SUB (ADD (input, sigma), sigma); input = SUB (input, split); devSplit[addry * lds + i] = split; devOutput[addry * lds + i] = input; if (max < fabs(input)) max = fabs(input); } shm[iTx] = max; __syncthreads (); #pragma unroll for (size_t i = nTx/2; i > 0; i>>=1) { if (iTx < i && shm[iTx] < shm[iTx+i]) shm[iTx] = shm[iTx+i]; __syncthreads (); } if (iTx == 0) devMax[addry] = shm[0]; } } __global__ void cuozblasSplitShiftMatEarlyExitNKernel ( const int32_t m, const int32_t n, const int32_t rho, const FP_CTYPE * __restrict__ devInput, const int32_t ldi, FP_CTYPE *devSplit, const int32_t lds, FP_CTYPE *devMax, const int32_t splitShift ) { const size_t iBx = blockIdx.x; const size_t nTx = SPLIT_N_NTX; const size_t nTy = SPLIT_N_NTY; const size_t iTx = threadIdx.x; const size_t iTy = threadIdx.y; const size_t addrx = iBx * nTx + iTx; FP_CTYPE sigma, max, input, split; if (addrx < m){ max = devMax[addrx]; sigma = scalbn (1., rho) * NextPowTwo (max) * splitShift; for (size_t j = iTy; j < n; j+=nTy) { input = devInput[j * ldi + addrx]; split = SUB (ADD (input, sigma), sigma); input = SUB (input, split); devSplit[j * lds + addrx] = split; } } } __global__ void cuozblasSplitShiftMatEarlyExitTKernel ( const int32_t m, const int32_t n, const int32_t rho, const FP_CTYPE * __restrict__ devInput, const int32_t ldi, FP_CTYPE *devSplit, const int32_t lds, FP_CTYPE *devMax, const int32_t splitShift ) { const size_t iBx = blockIdx.x; const size_t iBy = blockIdx.y; const size_t nTx = blockDim.x; const size_t nTy = blockDim.y; const size_t iTx = threadIdx.x; const size_t iTy = threadIdx.y; const size_t addrx = iBx * nTx + iTx; const size_t addry = iBy * nTy + iTy; FP_CTYPE sigma, max, input, split; if (addry < n){ max = devMax[addry]; sigma = scalbn (1., rho) * NextPowTwo (max) * splitShift; for (size_t i = addrx; i < m; i += nTx) { input = devInput[addry * ldi + i]; split = SUB (ADD (input, sigma), sigma); input = SUB (input, split); devSplit[addry * lds + i] = split; } } } __host__ void cuozblasSplitShiftMatDevice ( const char major, const int32_t m, const int32_t n, const FP_CTYPE *devInput, // input matrix (devAwrk) const int32_t ldi, // leading dimension of input matrix FP_CTYPE *devOutput, // output matrix (devAwrk) FP_CTYPE *devSplit, // split matrices (output): this includes NumSplitMax matrices const int32_t lds, // leading dimension of split matrix (# of cols) FP_CTYPE *devMax, uint32_t lastFlag, const int32_t splitShift ) { size_t ntx, nty, nbx, nby; dim3 threads, grid; const int32_t dim = ((major == 'r') && n > 1) ? n : m; int32_t rho = ceil((double)(UNIT+log2((double)dim))/2.); if (major == 'r') { if (n == 1) { ntx = SPLIT_VEC_NTX; threads = dim3 (ntx); if (!lastFlag) { nbx = min ((int)ceil((float)m/ntx), SPLIT_VEC_NBX); // 512 grid = dim3 (nbx); FP_CTYPE *devMaxOld = devMax; // because devMax may be overwritten by other thread-blocks devMaxOld = &devMax[getPitchSize(SPLIT_VEC_NBX)]; cudaMemcpy (devMaxOld, devMax, sizeof(FP_CTYPE), cudaMemcpyDeviceToDevice); cuozblasSplitShiftVecKernel <<< grid, threads, sizeof(FP_CTYPE) * ntx >>> (m, rho, devInput, devOutput, devSplit, devMaxOld, devMax, splitShift); grid = dim3 (1); cuozblasSplitVecSubKernel <<< grid, threads, sizeof(FP_CTYPE) * ntx >>> (nbx, devMax); } else { nbx = ceil((float)m/ntx); grid = dim3 (nbx); cuozblasSplitShiftVecEarlyExitKernel <<< grid, threads >>> (m, rho, devInput, devSplit, devMax, splitShift); } } else { // printf ("\ndevMaxA"); // PrintMat (n, 1, devMax, n); ntx = SPLIT_N_NTX; nty = SPLIT_N_NTY; nbx = ceil (float(m) / ntx); nby = 1; threads = dim3 (ntx, nty); grid = dim3 (nbx, nby); if (!lastFlag) cuozblasSplitShiftMatNKernel <<< grid, threads, sizeof(FP_CTYPE) * ntx * nty >>> (m, n, rho, devInput, ldi, devOutput, devSplit, lds, devMax, splitShift); else cuozblasSplitShiftMatEarlyExitNKernel <<< grid, threads >>> (m, n, rho, devInput, ldi, devSplit, lds, devMax, splitShift); } } else { ntx = SPLIT_T_NTX; nty = SPLIT_T_NTY; nbx = 1; nby = ceil (float(n) / nty); threads = dim3 (ntx, nty); grid = dim3 (nbx, nby); if (!lastFlag) cuozblasSplitShiftMatTKernel <<< grid, threads, sizeof(FP_CTYPE) * ntx * 2 >>> (m, n, rho, devInput, ldi, devOutput, devSplit, lds, devMax, splitShift); else cuozblasSplitShiftMatEarlyExitTKernel <<< grid, threads >>> (m, n, rho, devInput, ldi, devSplit, lds, devMax, splitShift); } } __host__ void cuozblasSplitShiftMat ( cuozblasHandle_t *oh, const char major, const int32_t m, const int32_t n, const FP_CTYPE *devInput, const int32_t ld, FP_CTYPE *devSplit, FP_CTYPE *devMax, const int32_t lds, uint32_t &numSplitArrays, uint32_t &lastFlag // note: numSplitArrays and lastFlag cannot be included in oh because this function is used for both row-split and col-split /* work: devSplit[0] input: devSplit[0] split: devSplit[i] */ ) { uint32_t numSplitArraysMax = numSplitArrays; uint32_t i = 1; FP_CTYPE check; // FindMax^(0) if (n == 1 && major == 'r') { int32_t ptrMax = 0; CuIAMAX (oh->ch, m, devInput, 1, &ptrMax); // Warning: note that cublas returns the index on the basis of 1-origin, while MKL (CBLAS) returns it on the basis of 0-origin !! cudaMemcpy (devMax, &devInput[ptrMax-1], sizeof(FP_CTYPE), cudaMemcpyDeviceToDevice); // so here do -1 !! } else { cuozblasFindMaxMatDevice (major, m, n, devInput, ld, devMax); } // Split^(0) & FindMax^(1) lastFlag = (oh->reproModeFlag && numSplitArraysMax == 1) ? 1 : 0; cuozblasSplitShiftMatDevice (major, m, n, devInput, ld, &devSplit[lds*n], &devSplit[0], lds, devMax, lastFlag, oh->splitShift); while (i < numSplitArraysMax) { // Check if (numSplitArraysMax-1 == i && !oh->reproModeFlag) { i++; break;} // when repromode=0 if (n == 1 && major == 'r') cudaMemcpy (&check, devMax, sizeof(FP_CTYPE), cudaMemcpyDeviceToHost); else CuASUM (oh->ch, (major=='r')?m:n, devMax, 1, &check); if (fabs(check) == 0) break; // Split^(i) & FindMax^(i+1) lastFlag = (oh->reproModeFlag && numSplitArraysMax-1 == i) ? 1 : 0; cuozblasSplitShiftMatDevice (major, m, n, &devSplit[lds*n*i], lds, &devSplit[lds*n*(i+1)], &devSplit[lds*n*i], lds, devMax, lastFlag, oh->splitShift); i++; } numSplitArrays = i; if (lastFlag == 1 && oh->numSplitArraysMax == 0) { fprintf (stderr, "* split error: out of memory (the result may be not correct rounding)\n"); exit (1); } } ozblas-1.2a17/src/cuda/cuozblas_split_sparse_shift.cu0000644000175200017520000001130613725177602023327 0ustar mukunokimukunoki// ========================================= // SplitMat // ========================================= __global__ void cuozblasSplitShiftMatSparseNKernel ( const int32_t m, const FP_CTYPE * __restrict__ devInput, const int32_t *devRowptr, FP_CTYPE *devOutput, FP_CTYPE *devSplit, FP_CTYPE *devMax, const int32_t splitShift ) { const size_t iBx = blockIdx.x; const size_t iTx = threadIdx.x; const size_t tid = blockDim.x * iBx + iTx; const size_t rowid = tid / 32; const size_t lane = tid % 32; FP_CTYPE sigma, max, input, split, tmp; __shared__ FP_CTYPE shm[512]; if (rowid < m){ int32_t dim = devRowptr[rowid+1] - devRowptr[rowid]; int32_t rho = ceil((double)(UNIT+log2((double)dim))/2.); max = devMax[rowid]; sigma = scalbn (1., rho) * NextPowTwo (max) / splitShift; max = 0.; for (size_t i = devRowptr[rowid] + lane; i < devRowptr[rowid+1]; i += 32) { input = devInput[i]; split = SUB (ADD (input, sigma), sigma); input = SUB (input, split); devSplit[i] = split; devOutput[i] = input; if (max < fabs(input)) max = fabs(input); } shm[(iTx/32)*32+lane] = max; __syncthreads (); if (lane == 0) { max = shm[(iTx/32)*32]; #pragma unroll for (size_t i = 1; i < 32; i++) { tmp = shm[(iTx/32)*32+i]; if (max < fabs(tmp)) max = fabs(tmp); } devMax[rowid] = max; } } } __global__ void cuozblasSplitShiftMatSparseEarlyExitNKernel ( const int32_t m, const FP_CTYPE * __restrict__ devInput, const int32_t * __restrict__ devRowptr, FP_CTYPE *devSplit, FP_CTYPE *devMax, const int32_t splitShift ) { const size_t iBx = blockIdx.x; const size_t iTx = threadIdx.x; const size_t tid = blockDim.x * iBx + iTx; const size_t rowid = tid / 32; const size_t lane = tid % 32; FP_CTYPE sigma, max, input, split; if (rowid < m){ int32_t dim = devRowptr[rowid+1] - devRowptr[rowid]; int32_t rho = ceil((double)(UNIT+log2((double)dim))/2.); max = devMax[rowid]; sigma = scalbn (1., rho) * NextPowTwo (max) / splitShift; for (size_t i = devRowptr[rowid] + lane; i < devRowptr[rowid+1]; i += 32) { input = devInput[i]; split = SUB (ADD (input, sigma), sigma); input = SUB (input, split); devSplit[i] = split; } } } __host__ void cuozblasSplitShiftMatSparseDevice ( const char trans, const int32_t m, const FP_CTYPE *devInput, // input matrix (devAwrk) const int32_t *devRowptr, FP_CTYPE *devOutput, // output matrix (devAwrk) FP_CTYPE *devSplit, // split matrices (output): this includes NumSplitMax matrices FP_CTYPE *devMax, uint32_t lastFlag, const int32_t splitShift ) { size_t ntx, nbx; dim3 threads, grid; /* FP_CTYPE *hstC, *devC; cudaMallocHost ((void **) &hstC, sizeof(FP_CTYPE) * m); cudaMalloc ((void **) &devC, sizeof(FP_CTYPE) * m); */ if (trans == 'n' || trans == 'N') { //PrintMat (m, 1, devMax, m); ntx = 512; nbx = ceil (float(m) / (ntx / 32.)); threads = dim3 (ntx); grid = dim3 (nbx); if (!lastFlag) cuozblasSplitShiftMatSparseNKernel <<< grid, threads >>> (m, devInput, devRowptr, devOutput, devSplit, devMax, splitShift); else cuozblasSplitShiftMatSparseEarlyExitNKernel <<< grid, threads >>> (m, devInput, devRowptr, devSplit, devMax, splitShift); } else { fprintf (stderr, "error: Split T is not implemented.\n"); exit (1); } /* cudaMemcpy (hstC, devC, sizeof(FP_CTYPE) * m, cudaMemcpyDeviceToHost); for (int32_t i = 0; i < m; i++) printf ("%d:%a\n", i, hstC[i]); cudaFree (devC); cudaFreeHost (hstC); */ } __host__ void cuozblasSplitShiftMatSparse ( cuozblasHandle_t *oh, const char trans, const int32_t m, const FP_CTYPE *devInput, const int32_t *devRowptr, FP_CTYPE *devSplit, FP_CTYPE *devMax, const int32_t lds, uint32_t &numSplitArrays, uint32_t &lastFlag ) { uint32_t numSplitArraysMax = numSplitArrays; uint32_t i = 1; FP_CTYPE check; // FindMax^(0) cuozblasFindMaxMatSparseDevice (trans, m, devInput, devRowptr, devMax); // Split^(0) & FindMax^(1) lastFlag = (oh->reproModeFlag && numSplitArraysMax == 1) ? 1 : 0; cuozblasSplitShiftMatSparseDevice (trans, m, devInput, devRowptr, &devSplit[lds], &devSplit[0], devMax, lastFlag, oh->splitShift); while (i < numSplitArraysMax) { // Check if (numSplitArraysMax-1 == i && !oh->reproModeFlag) { i++; break;} // when repromode=0 CuASUM (oh->ch, m, devMax, 1, &check); if (fabs(check) == 0) break; // Split^(i) & FindMax^(i+1) lastFlag = (oh->reproModeFlag && numSplitArraysMax-1 == i) ? 1 : 0; cuozblasSplitShiftMatSparseDevice (trans, m, &devSplit[lds*i], devRowptr, &devSplit[lds*(i+1)], &devSplit[lds*i], devMax, lastFlag, oh->splitShift); i++; } numSplitArrays = i; if (lastFlag == 1 && oh->numSplitArraysMax == 0) { fprintf (stderr, "split error: out of memory (the result may be not correct rounding)\n"); exit (1); } } ozblas-1.2a17/src/cuda/cuozblas_csrmv.cu0000644000175200017520000002031413725177602020553 0ustar mukunokimukunoki#include "cuozblas_common.h" __host__ int32_t OzCSRMV ( cuozblasHandle_t *oh, const cusparseOperation_t tranA, const int32_t m, const int32_t n, const int32_t nnz, const FP_DTYPE *alpha, const cusparseMatDescr_t descrA, const FP_DTYPE *devA, const int32_t *devArowptr, const int32_t *devAcolind, const FP_DTYPE *devB, const FP_DTYPE *beta, FP_DTYPE *devC ) { counterInit (oh); if (oh->reproModeFlag == 0 && oh->numSplitArraysMax == 1) { double t0 = timer(); #if defined (PREC_SD) printf ("not supported\n"); exit(1); #else RCuCSRMV (oh->csh, tranA, m, n, nnz, alpha, descrA, devA, devArowptr, devAcolind, devB, beta, devC); #endif oh->t_SpMV_SpMM_total += timer() - t0; return 0; } cusparsePointerMode_t pmode; cusparseGetPointerMode(oh->csh, &pmode); cusparseSetPointerMode(oh->csh, CUSPARSE_POINTER_MODE_HOST); FP_CTYPE fone = 1., fzero = 0.; double t1, t0 = timer(); uint32_t numSplitArraysMax = (oh->numSplitArraysMax > 0) ? oh->numSplitArraysMax : NumSplitDefaultMax; int32_t ldas, ldbs, ldcs; FP_CTYPE *devASplit; FP_CTYPE *devAmax, *devBmax; FP_CTYPE *devBSplit, *devCSplit; int32_t mbk = m; // currently blocking is not supported if (mbk != m && oh->memMaskSplitA) { // when blocking is applied, memMaskSplitA cannot be used printf ("Blocking is not supported with WithOutSplitA mode\n"); exit(1); } if (oh->memMaskSplitA != 0) oh->memAddr = 0; cuozblasMallocSplitVec (oh, nnz, numSplitArraysMax, &devASplit, &devAmax, ldas); // this is needed even when memMaskSplitA if (oh->memMaskSplitA != 0) oh->memMaskSplitA = oh->memAddr; cuozblasMallocSplitVec (oh, n, numSplitArraysMax, &devBSplit, &devBmax, ldbs); ldcs = getPitchSize (mbk); devCSplit = (FP_CTYPE*)(oh->devWork + oh->memAddr); oh->memAddr += sizeof(FP_CTYPE) * ldcs * numSplitArraysMax * numSplitArraysMax; #if defined (PREC_SD) int32_t ldat, ldbt; FP_CTYPE *devATmp, *devBTmp; if (oh->memMaskSplitA == 0) { cuozblasSDMallocTmpVec (oh, nnz, &devATmp, ldat); cuozblasSDCopyMat (nnz, 1, devA, nnz, devATmp, ldat); } cuozblasSDMallocTmpVec (oh, n, &devBTmp, ldbt); cuozblasSDCopyMat (n, 1, devB, n, devBTmp, ldbt); #endif memCheck (oh); for (int32_t im = 0; im < ceil((float)m/mbk); im++) { int32_t mbk_ = (m-mbk*im >= mbk) ? mbk : m-mbk*im; // Split of A ----------------------------------- t1 = timer(); if (oh->memMaskSplitA == 0) { oh->numSplitArraysA = numSplitArraysMax; #if defined (PREC_SD) cuozblasSplitShiftMatSparse (oh, 'n', m, devATmp, devArowptr, devASplit, devAmax, ldas, oh->numSplitArraysA, oh->lastFlagA); #else cuozblasSplitShiftMatSparse (oh, 'n', m, devA, devArowptr, devASplit, devAmax, ldas, oh->numSplitArraysA, oh->lastFlagA); #endif } else { devASplit = (FP_CTYPE*)devA; oh->numSplitArraysA = oh->numSplitArraysA_; } oh->t_SplitA += timer() - t1; // Split of B ----------------------------------- t1 = timer(); oh->numSplitArraysB = numSplitArraysMax; #if defined (PREC_SD) cuozblasSplitShiftMat (oh, 'r', n, 1, devBTmp, ldbt, devBSplit, devBmax, ldbs, oh->numSplitArraysB, oh->lastFlagB); #else cuozblasSplitShiftMat (oh, 'r', n, 1, devB, ldbs, devBSplit, devBmax, ldbs, oh->numSplitArraysB, oh->lastFlagB); #endif oh->t_SplitB += timer() - t1; // Compute -------------------------------------- t1 = timer(); uint32_t numB, ia, ic = 0; uint32_t maxlevel = (oh->numSplitArraysA-1) + (oh->numSplitArraysB-1); if (oh->fastModeFlag) { if (oh->numSplitArraysMax == 0) maxlevel = min (oh->numSplitArraysA-1, oh->numSplitArraysB-1); else maxlevel = oh->numSplitArraysMax-1; } FP_CTYPE *ptrA, *ptrB, *ptrC; ptrC = devCSplit; for (ia = 0; ia < min (maxlevel+1, oh->numSplitArraysA); ia++) { ptrA = devASplit+ldas*ia; if (oh->useGemmFlag) { numB = min (oh->numSplitArraysB, maxlevel+1 - ia); ptrB = devBSplit; CuCSRMM (oh->csh, tranA, mbk_, numB, n, nnz, &fone, descrA, ptrA, devArowptr, devAcolind, ptrB, ldbs, &fzero, ptrC, ldcs); ptrC += ldcs*numB; ic += numB; } else { for (uint32_t ib = 0; ib < oh->numSplitArraysB; ib++) { if (ia + ib <= maxlevel) { ptrB = devBSplit+ldbs*ib; ptrC = devCSplit+ldcs*ic; CuCSRMV (oh->csh, tranA, mbk_, n, nnz, &fone, descrA, ptrA, devArowptr, devAcolind, ptrB, &fzero, ptrC); ic++; } } } } oh->numSplitArraysC = ic; oh->t_comp += timer() - t1; // Sum ----------------------------------------- t1 = timer(); cublasPointerMode_t pmode_ = (pmode == CUSPARSE_POINTER_MODE_HOST) ? CUBLAS_POINTER_MODE_HOST : CUBLAS_POINTER_MODE_DEVICE; cuozblasSumMat (mbk_, 1, devCSplit, ldcs, ldcs, oh->numSplitArraysC, &devC[im*mbk], ldcs, alpha, beta, pmode_, oh->sumModeFlag); oh->t_sum += timer() - t1; } oh->t_total = timer() - t0; // for CG, time // ================================= oh->t_SplitMat_total += oh->t_SplitA; oh->t_SplitVec_total += oh->t_SplitB; oh->t_Sum_total += oh->t_sum; oh->t_AXPY_SCAL_total += 0.; oh->t_DOT_NRM2_total += 0.; oh->t_SpMV_SpMM_total += oh->t_comp; // ================================= cusparseSetPointerMode(oh->csh, pmode); return 0; } __host__ int32_t OzCSRMVSplitA ( cuozblasHandle_t *oh, const cusparseOperation_t tranA, const int32_t m, const int32_t n, const int32_t nnz, const cusparseMatDescr_t descrA, const FP_DTYPE *devA, const int32_t *devArowptr, const int32_t *devAcolind, FP_CTYPE **devASplit ) { counterInit (oh); uint32_t numSplitArraysMax = (oh->numSplitArraysMax > 0) ? oh->numSplitArraysMax : NumSplitDefaultMax; uint32_t numSplitArraysMaxOrig = oh->numSplitArraysMax; int32_t ldas; FP_CTYPE *devAmax; #if defined (PREC_SD) int32_t ldat; FP_CTYPE *devATmp; cuozblasSDMallocTmpVec (oh, nnz, &devATmp, ldat); cuozblasSDCopyMat (nnz, 1, devA, nnz, devATmp, ldat); #endif cuozblasMallocSplitMatSparse (oh, m, nnz, numSplitArraysMax, &devASplit[0], &devAmax, ldas); memCheck (oh); oh->memMaskSplitA = oh->memAddr; double t1 = timer(); if (oh->reproModeFlag == 0 && oh->numSplitArraysMax == 1) { cudaMemcpy (devASplit[0], devA, sizeof(FP_CTYPE)*nnz, cudaMemcpyDeviceToDevice); } else { // Split of A ----------------------------------- oh->numSplitArraysA = numSplitArraysMax; #if defined (PREC_SD) cuozblasSplitMatSparse (oh, 'n', m, devATmp, devArowptr, devASplit[0], devAmax, ldas, oh->numSplitArraysA, oh->lastFlagA); #else cuozblasSplitMatSparse (oh, 'n', m, devA, devArowptr, devASplit[0], devAmax, ldas, oh->numSplitArraysA, oh->lastFlagA); #endif // shiftSize-tuning // ------------------------------------------------------------------------ if (numSplitArraysMaxOrig == 0) { // tuning is possible only when full-splitting (d=0) printf ("## CSRMV: << shift-size tuning >> num.split = %d -> ", oh->numSplitArraysA); oh->splitShift = 1; int32_t numSplitArraysAOld = oh->numSplitArraysA; do { numSplitArraysAOld = oh->numSplitArraysA; oh->splitShift *= 2; // try with new shift-size oh->numSplitArraysA = numSplitArraysMax; #if defined (PREC_SD) cuozblasSplitShiftMatSparse (oh, 'n', m, devATmp, devArowptr, devASplit[0], devAmax, ldas, oh->numSplitArraysA, oh->lastFlagA); #else cuozblasSplitShiftMatSparse (oh, 'n', m, devA, devArowptr, devASplit[0], devAmax, ldas, oh->numSplitArraysA, oh->lastFlagA); #endif // if numSplit increased, stop } while (numSplitArraysAOld == oh->numSplitArraysA); // do again with the optimal shift-size // oh->splitShift /= 2; oh->numSplitArraysA = numSplitArraysMax; #if defined (PREC_SD) cuozblasSplitShiftMatSparse (oh, 'n', m, devATmp, devArowptr, devASplit[0], devAmax, ldas, oh->numSplitArraysA, oh->lastFlagA); #else cuozblasSplitShiftMatSparse (oh, 'n', m, devA, devArowptr, devASplit[0], devAmax, ldas, oh->numSplitArraysA, oh->lastFlagA); #endif printf ("%d (with splitShift = %d)\n", oh->numSplitArraysA, oh->splitShift); } // ------------------------------------------------------------------------ oh->numSplitArraysA_ = oh->numSplitArraysA; } oh->t_SplitA += timer() - t1; // for CG, time // ================================= oh->t_SplitMat_total += oh->t_SplitA; oh->t_SplitVec_total += 0.; oh->t_Sum_total += 0.; oh->t_AXPY_SCAL_total += 0.; oh->t_DOT_NRM2_total += 0.; oh->t_SpMV_SpMM_total += 0.; // ================================= return 0; } ozblas-1.2a17/src/cuda/cuozblas_cg.cu0000644000175200017520000001612713725200232020003 0ustar mukunokimukunoki#include "cuozblas_common.h" // true residual is computed using standard FP operations FP_DTYPE getTrueResidual ( cuozblasHandle_t *oh, const cusparseOperation_t tranA, const int32_t dimN, const int32_t dimNNZ, const cusparseMatDescr_t descrA, const FP_DTYPE *matA, const int32_t *matArowptr, const int32_t *matAcolind, const FP_DTYPE *vecB, const FP_DTYPE *vecX, FP_DTYPE *vecT ) { FP_DTYPE res0, fone = 1., fnegone = -1.; // residual: r_i = b-Ax_i cudaMemcpy (vecT, vecB, sizeof(FP_DTYPE) * dimN, cudaMemcpyDeviceToDevice); // t = b RCuCSRMV (oh->csh, tranA, dimN, dimN, dimNNZ, &fnegone, descrA, matA, matArowptr, matAcolind, vecX, &fone, vecT); // t = t-Ax (b-Ax) RCuNRM2 (oh->ch, dimN, vecT, 1, &res0); return res0; } __host__ int32_t OzCG ( cuozblasHandle_t *oh, const cusparseOperation_t tranA, const int32_t dimN, const int32_t dimNNZ, const cusparseMatDescr_t descrA, const FP_DTYPE *matA, const int32_t *matArowptr, const int32_t *matAcolind, const FP_DTYPE *vecB, FP_DTYPE *vecX, int32_t maxiter, double tol ) { // ================================= oh->t_SplitMat_total = 0.; oh->t_SplitVec_total = 0.; oh->t_Sum_total = 0.; oh->t_AXPY_SCAL_total = 0.; oh->t_DOT_NRM2_total = 0.; oh->t_SpMV_SpMM_total = 0.; // ================================= int32_t numiter; double t0, t1, t2, t3; float *dat1, *dat2; if (oh->verbose > 0) { dat1 = (float *)malloc(sizeof(float)*maxiter/oh->verbose+1); dat2 = (float *)malloc(sizeof(float)*maxiter/oh->verbose+1); } t1 = timer(); // ----------------------------------------------------- FP_DTYPE alpha, beta, negalpha, resi, res0, nrmb, tmp, dnew, dold; FP_DTYPE fzero = 0., fone = 1., fnegone = -1.; FP_DTYPE *vecT, *vecR, *vecP, *vecQ; cudaMalloc ((void **) &vecT, sizeof(FP_DTYPE) * dimN); cudaMalloc ((void **) &vecR, sizeof(FP_DTYPE) * dimN); cudaMalloc ((void **) &vecP, sizeof(FP_DTYPE) * dimN); cudaMalloc ((void **) &vecQ, sizeof(FP_DTYPE) * dimN); cublasSetPointerMode(oh->ch, CUBLAS_POINTER_MODE_HOST); #if defined (PREC_SD) printf ("## * On PREC_SD mat-split is performed at each iterration\n"); float *matASplit; matASplit = (float*)matA; #else FP_DTYPE *matASplit; OzCSRMVSplitA (oh, tranA, dimN, dimN, dimNNZ, descrA, matA, matArowptr, matAcolind, &matASplit); #endif OzNRM2 (oh, dimN, vecB, 1, &nrmb); // nrmb = |b| // residual: r_0 = b-Ax_0 cudaMemcpy (vecR, vecB, sizeof(FP_DTYPE) * dimN, cudaMemcpyDeviceToDevice); // r = b OzCSRMV (oh, tranA, dimN, dimN, dimNNZ, &fnegone, descrA, matASplit, matArowptr, matAcolind, vecX, &fone, vecR); // r = r-Ax (b-Ax) OzDOT (oh, dimN, vecR, 1, vecR, 1, &dold); // dold = resi = sqrt (dold); // resi = |r| if (oh->verbose > 0) { t3 = timer(); dat1[0] = resi/nrmb; dat2[0] = (oh->trueres==0) ? t3-t1 : getTrueResidual (oh, tranA, dimN, dimNNZ, descrA, matA, matArowptr, matAcolind, vecB, vecX, vecT) / nrmb; } cudaMemcpy (vecP, vecR, sizeof(FP_DTYPE) * dimN, cudaMemcpyDeviceToDevice); // p = r numiter = 0; t2 = timer(); // ----------------------------------------------------- while (numiter < maxiter) { numiter++; OzCSRMV (oh, tranA, dimN, dimN, dimNNZ, &fone, descrA, matASplit, matArowptr, matAcolind, vecP, &fzero, vecQ); // q = Ap OzDOT (oh, dimN, vecP, 1, vecQ, 1, &tmp); // tmp = alpha = dold / tmp; negalpha = -alpha; t0 = timer(); OzAXPY (oh, dimN, &alpha, vecP, 1, vecX, 1); // x = x+alpha*p OzAXPY (oh, dimN, &negalpha, vecQ, 1, vecR, 1); // r = r-alpha*q oh->t_AXPY_SCAL_total += timer() - t0; OzDOT (oh, dimN, vecR, 1, vecR, 1, &dnew); // dnew = resi = sqrt(dnew); beta = dnew / dold; // beta = dnew/dold dold = dnew; t0 = timer(); CuSCAL (oh->ch, dimN, &beta, vecP, 1); // p = beta*p OzAXPY (oh, dimN, &fone, vecR, 1, vecP, 1); // p = p+r oh->t_AXPY_SCAL_total += timer() - t0; if (oh->verbose > 0) { if (numiter%oh->verbose == 0 ) { t3 = timer(); dat1[numiter/oh->verbose] = resi/nrmb; dat2[numiter/oh->verbose] = (oh->trueres==0) ? t3-t1 : getTrueResidual (oh, tranA, dimN, dimNNZ, descrA, matA, matArowptr, matAcolind, vecB, vecX, vecT) / nrmb; } } if (resi/nrmb < tol) break; } t3 = timer(); // ----------------------------------------------------- // ================================= double tloc_SpMV_SpMM_total = oh->t_SpMV_SpMM_total; double tloc_DOT_NRM2_total = oh->t_DOT_NRM2_total; double tloc_AXPY_SCAL_total = oh->t_AXPY_SCAL_total; double tloc_Sum_total = oh->t_Sum_total; double tloc_SplitVec_total = oh->t_SplitVec_total; double tloc_SplitMat_total = oh->t_SplitMat_total; double tloc_Other_total = (t3-t1) -tloc_SpMV_SpMM_total -tloc_DOT_NRM2_total -tloc_AXPY_SCAL_total -tloc_Sum_total -tloc_SplitVec_total -tloc_SplitMat_total; // ================================= if (oh->verbose > 0) { // residual: r_i = b-Ax_i (correctly-rounded) cudaMemcpy (vecT, vecB, sizeof(FP_DTYPE) * dimN, cudaMemcpyDeviceToDevice); // t = b oh->memMaskSplitA = 0; // this is needed oh->numSplitArraysMax = 0; oh->reproModeFlag = 1; oh->fastModeFlag = 0; oh->sumModeFlag = 1; OzCSRMV (oh, tranA, dimN, dimN, dimNNZ, &fnegone, descrA, matA, matArowptr, matAcolind, vecX, &fone, vecT); // t = t-Ax (b-Ax) OzNRM2 (oh, dimN, vecT, 1, &res0); // note: parameter-reset is needed here if user wants to continue the use of OzBLAS later on if (oh->trueres) printf ("\n## iter\t||r_i||/||b||\t||b-Ax||/||b||\n"); else printf ("\n## iter\t||r_i||/||b||\ttime(sec)\n"); for (int32_t i = 1; i <= numiter/oh->verbose; i++) printf ("%d\t%1.3e\t%1.3e\n", i*oh->verbose, dat1[i], dat2[i]); printf ("## total time = %1.3e\n", t3-t1); if (oh->trueres) printf ("## -> including true residual computation\n"); printf ("## iter-part time = %1.3e\n", t3-t2); if (oh->trueres) printf ("## -> including true residual computation\n"); printf ("## total iter = %d\n", numiter); printf ("## time per iter = %1.3e\n", (t3-t2)/numiter); printf ("## ||r_i||/||b|| = %1.3e\n", resi/nrmb); printf ("## ||b-Ax||/||b|| = %1.3e\n", res0/nrmb); // ================================= printf ("%1.3e\t", tloc_SpMV_SpMM_total); printf ("%1.3e\t", tloc_DOT_NRM2_total); printf ("%1.3e\t", tloc_AXPY_SCAL_total); printf ("%1.3e\t", tloc_Sum_total); printf ("%1.3e\t", tloc_SplitVec_total); printf ("%1.3e\t", tloc_SplitMat_total); printf ("%1.3e\t#(SpMV/MM, DOT/NRM, AXPY/SCAL, Sum, SpltVec, SpltMat, Other)\n", tloc_Other_total); // ================================= printf ("%d\t%1.3e\t(%a)\t%1.3e\t(%a)\t%1.3e\t%1.3e\t#(iter, res, trueres, ttl-time, itr-prt-time)\n", numiter, resi/nrmb, resi/nrmb, res0/nrmb, res0/nrmb, t3-t1, t3-t2); free (dat1); free (dat2); } cudaFree (vecT); cudaFree (vecR); cudaFree (vecP); cudaFree (vecQ); // ================================= oh->t_SplitMat_total = 0.; oh->t_SplitVec_total = 0.; oh->t_Sum_total = 0.; oh->t_AXPY_SCAL_total = 0.; oh->t_DOT_NRM2_total = 0.; oh->t_SpMV_SpMM_total = 0.; // reset parameters oh->memMaskSplitA = 0; // disable pre-split of matA oh->splitShift = 1; // default (no-Splitshift) // ================================= return 0; } ozblas-1.2a17/src/cpu/0000755000175200017520000000000013736552053015040 5ustar mukunokimukunokiozblas-1.2a17/src/cpu/ozblas_gemm.cpp0000644000175200017520000001504413725177602020050 0ustar mukunokimukunoki#include "ozblas_common.h" #if !defined (PREC_SD) int OzGEMM ( ozblasHandle_t *oh, const char tranA, const char tranB, const int m, const int n, const int k, const FP_DTYPE alpha, const FP_DTYPE *devA, const int lda, const FP_DTYPE *devB, const int ldb, const FP_DTYPE beta, FP_DTYPE *devC, const int ldc ) { counterInit (oh); if (oh->reproModeFlag == 0 && oh->numSplitArraysMax == 1) { #if (NVBLAS) RNVBLAS_GEMM (&tranA, &tranB, &m, &n, &k, &alpha, devA, &lda, devB, &ldb, &beta, devC, &ldc); #else RCBLAS_GEMM (CblasColMajor, ToCblasOp(tranA), ToCblasOp(tranB), m, n, k, alpha, devA, lda, devB, ldb, beta, devC, ldc); #endif return 0; } FP_CTYPE fone = 1., fzero = 0.; double t1, t0 = timer(); uint32_t numSplitArraysMax = (oh->numSplitArraysMax > 0) ? oh->numSplitArraysMax : NumSplitDefaultMax; int ldas, ldbs, ldcs; FP_CTYPE *devAmax, *devBmax; FP_CTYPE *devASplit, *devBSplit, *devCSplit; int mbk = m; int nbk = n; uint64_t memAddrTmp = oh->memAddr; while (mbk > 0) { ozblasMallocSplitMat (oh, k, mbk, numSplitArraysMax, &devASplit, &devAmax, ldas); // TRANSPOSE ozblasMallocSplitMat (oh, k, nbk, numSplitArraysMax, &devBSplit, &devBmax, ldbs); ozblasMallocResultMat (oh, mbk, n, numSplitArraysMax, &devCSplit, ldcs); if (!memCheck (oh)) break; oh->memAddr = memAddrTmp; mbk = ceil (mbk / 2.); nbk = ceil (nbk / 2.); } FP_CTYPE **devAptr, **devBptr, **devCptr; if (oh->useBatchedGemmFlag) { int memorysize = 0; devAptr = (FP_CTYPE**)(oh->devBatchAddr); memorysize += sizeof(FP_CTYPE*) * numSplitArraysMax * numSplitArraysMax; devBptr = (FP_CTYPE**)(oh->devBatchAddr + memorysize); memorysize += sizeof(FP_CTYPE*) * numSplitArraysMax * numSplitArraysMax; devCptr = (FP_CTYPE**)(oh->devBatchAddr + memorysize); memorysize = 0; devAptr = (FP_CTYPE**)(oh->devBatchAddr); memorysize += sizeof(FP_CTYPE*) * numSplitArraysMax * numSplitArraysMax; devBptr = (FP_CTYPE**)(oh->devBatchAddr + memorysize); memorysize += sizeof(FP_CTYPE*) * numSplitArraysMax * numSplitArraysMax; devCptr = (FP_CTYPE**)(oh->devBatchAddr + memorysize); } for (int im = 0; im < ceil((float)m/mbk); im++) { int mbk_ = (m-mbk*im >= mbk) ? mbk : m-mbk*im; // Split of A ----------------------------------- t1 = timer(); oh->numSplitArraysA = numSplitArraysMax; if (tranA == 'n' || tranA == 'N') { #ifdef MKL MKL_OMATCOPY ('c', 't', k, mbk_, fone, devA+im*mbk, lda, devASplit, ldas); // TRANSPOSE #else CBLAS_OMATCOPY (CblasColMajor, CblasTrans, mbk_, k, fone, devA+im*mbk, lda, devASplit, ldas); // TRANSPOSE #endif ozblasSplitMat (oh, 'c', k, mbk_, devASplit, ldas, devASplit, devAmax, ldas, oh->numSplitArraysA, oh->lastFlagA); // TRANSPOSE } else { ozblasSplitMat (oh, 'c', k, mbk_, devA+im*mbk, lda, devASplit, devAmax, ldas, oh->numSplitArraysA, oh->lastFlagA); } oh->t_SplitA += timer() - t1; for (int in = 0; in < ceil((float)n/nbk); in++) { int nbk_ = (n-nbk*in >= nbk) ? nbk : n-nbk*in; // Split of B ----------------------------------- t1 = timer(); oh->numSplitArraysB = numSplitArraysMax; if (tranB == 'n' || tranB == 'N') { ozblasSplitMat (oh, 'c', k, nbk_, devB+in*nbk*ldb, ldb, devBSplit, devBmax, ldbs, oh->numSplitArraysB, oh->lastFlagB); } else { #ifdef MKL MKL_OMATCOPY ('c', 't', k, nbk_, fone, devB+in*nbk*ldb, ldb, devBSplit, ldbs); // TRANSPOSE #else CBLAS_OMATCOPY (CblasColMajor, CblasTrans, nbk_, k, fone, devB+in*nbk*ldb, ldb, devBSplit, ldbs); // TRANSPOSE #endif ozblasSplitMat (oh, 'c', k, nbk_, devBSplit, ldbs, devBSplit, devBmax, ldbs, oh->numSplitArraysB, oh->lastFlagB); } oh->t_SplitB += timer() - t1; // Compute -------------------------------------- uint32_t ia, ib, ic; uint32_t maxlevel = (oh->numSplitArraysA-1) + (oh->numSplitArraysB-1); if (oh->fastModeFlag) { if (oh->numSplitArraysMax == 0) maxlevel = MIN (oh->numSplitArraysA-1, oh->numSplitArraysB-1); else maxlevel = oh->numSplitArraysMax-1; } if (oh->numSplitArraysMax == 1) { t1 = timer(); #if (NVBLAS) char tranAt = 't'; NVBLAS_GEMM (&tranAt, &tranB, &mbk_, &nbk_, &k, &fone, devASplit, &ldas, devBSplit, &ldbs, &fzero, devCSplit, &ldcs); #else CBLAS_GEMM (CblasColMajor, CblasTrans, ToCblasOp(tranB), mbk_, nbk_, k, fone, devASplit, ldas, devBSplit, ldbs, fzero, devCSplit, ldcs); #endif oh->numSplitArraysC = 1; } else { ic = 0; if (oh->useBatchedGemmFlag) { for (ia = 0; ia < oh->numSplitArraysA; ia++) { for (ib = 0; ib < oh->numSplitArraysB; ib++) { if (ia + ib <= maxlevel) { devAptr[ic] = devASplit+ldas*mbk*ia; devBptr[ic] = devBSplit+ldbs*nbk*ib; devCptr[ic] = devCSplit+ldcs*nbk*ic; ic++; } } } oh->numSplitArraysC = ic; t1 = timer(); #ifdef MKL #if (NVBLAS) char tranAt = 't'; char tranBn = 'n'; NVBLAS_GEMMBATCH (&tranAt, &tranBn, &mbk_, &nbk_, &k, &fone, (const FP_CTYPE**)devAptr, &ldas, (const FP_CTYPE**)devBptr, &ldbs, &fzero, (FP_CTYPE**)devCptr, &ldcs, &one, (int*)&oh->numSplitArraysC); #else CBLAS_TRANSPOSE trans_ = CblasTrans; CBLAS_TRANSPOSE notrans_ = CblasNoTrans; CBLAS_GEMMBATCH (CblasColMajor, &trans_, ¬rans_, &mbk_, &nbk_, &k, &fone, (const FP_CTYPE**)devAptr, &ldas, (const FP_CTYPE**)devBptr, &ldbs, &fzero, (FP_CTYPE**)devCptr, &ldcs, 1, (int*)&oh->numSplitArraysC); #endif #else fprintf (stderr, "* no batched BLAS support.\n"); exit(1); #endif } else { t1 = timer(); FP_CTYPE *ptrA, *ptrB, *ptrC; for (ia = 0; ia < oh->numSplitArraysA; ia++) { for (ib = 0; ib < oh->numSplitArraysB; ib++) { if (ia + ib <= maxlevel) { ptrA = devASplit+ldas*mbk*ia; ptrB = devBSplit+ldbs*nbk*ib; ptrC = devCSplit+ldcs*nbk*ic; #if (NVBLAS) char tranAt = 't'; char tranBn = 'n'; NVBLAS_GEMM (&tranAt, &tranBn, &mbk_, &nbk_, &k, &fone, ptrA, &ldas, ptrB, &ldbs, &fzero, ptrC, &ldcs); #else CBLAS_GEMM (CblasColMajor, CblasTrans, CblasNoTrans, mbk_, nbk_, k, fone, ptrA, ldas, ptrB, ldbs, fzero, ptrC, ldcs); #endif ic++; } } } oh->numSplitArraysC = ic; } } oh->t_comp += timer() - t1; // Sum ----------------------------------------- t1 = timer(); ozblasSumMat (mbk_, nbk_, devCSplit, ldcs*nbk, ldcs, oh->numSplitArraysC, &devC[ldc*(in*nbk)+im*mbk], ldc, alpha, beta, oh->sumModeFlag); oh->t_sum += timer() - t1; } } oh->t_total = timer() - t0; return 0; } #endif ozblas-1.2a17/src/cpu/ozblas_nrm2.cpp0000644000175200017520000000114613725177602017777 0ustar mukunokimukunoki#include "ozblas_common.h" // reproducible but not accurate FP_DTYPE OzNRM2 ( ozblasHandle_t *oh, const int n, const FP_DTYPE* devX, const int incx ) { counterInit (oh); if (oh->reproModeFlag == 0 && oh->numSplitArraysMax == 1) { return RCBLAS_NRM2 (n, devX, incx); } if (incx != 1) { fprintf (stderr, "* incx and incy are not supported.\n"); exit (1); } FP_DTYPE ret; ret = OzDOT (oh, n, devX, incx, devX, incx); // ------------------------------ // computation of SQRT (ret) on host // Not accurate but reproducible ret = sqrt (ret); // ------------------------------ return ret; } ozblas-1.2a17/src/cpu/ozblas_gemv.cpp0000644000175200017520000001222713725177602020061 0ustar mukunokimukunoki#include "ozblas_common.h" int OzGEMV ( ozblasHandle_t *oh, const char tranA, const int m, const int n, const FP_DTYPE alpha, const FP_DTYPE *devA, const int lda, const FP_DTYPE *devB, const int incx, const FP_DTYPE beta, FP_DTYPE *devC, const int incy ) { counterInit (oh); #if defined (PREC_SD) fprintf (stderr, "* PREC_SD is not implemented.\n"); exit (1); #else if (oh->reproModeFlag == 0 && oh->numSplitArraysMax == 1) { RCBLAS_GEMV (CblasColMajor, ToCblasOp(tranA), m, n, alpha, devA, lda, devB, incx, beta, devC, incy); return 0; } if (incx != 1 || incy != 1 ) { fprintf (stderr, "* incx and incy are not supported.\n"); exit (1); } FP_CTYPE fone = 1., fzero = 0.; int ione = 1; double t1, t0 = timer(); uint32_t numSplitArraysMax = (oh->numSplitArraysMax > 0) ? oh->numSplitArraysMax : NumSplitDefaultMax; int ldas, ldbs, ldcs; FP_CTYPE *devASplit; FP_CTYPE *devAmax, *devBmax; FP_CTYPE *devBSplit, *devCSplit; int mbk = m; int memAddrTmp = oh->memAddr; #if defined (PREC_SD) int ldat, ldbt; FP_CTYPE *devATmp, *devBTmp; #endif while (mbk > 0) { if (tranA == 'N' || tranA == 'n') { ozblasMallocSplitMat (oh, n, mbk, numSplitArraysMax, &devASplit, &devAmax, ldas); // !! TRANSPOSE !! ozblasMallocSplitVec (oh, n, numSplitArraysMax, &devBSplit, &devBmax, ldbs); ozblasMallocResultVec (oh, mbk, numSplitArraysMax, &devCSplit, ldcs); } else { ozblasMallocSplitMat (oh, mbk, n, numSplitArraysMax, &devASplit, &devAmax, ldas); ozblasMallocSplitVec (oh, mbk, numSplitArraysMax, &devBSplit, &devBmax, ldbs); ozblasMallocResultVec (oh, n, numSplitArraysMax, &devCSplit, ldcs); } #if defined (PREC_SD) if (tranA == 'N' || tranA == 'n') { ozblasSDMallocTmpMat (oh, n, mbk, &devATmp, ldat); // transposed ozblasSDMallocTmpMat (oh, n, 1, &devBTmp, ldbt); ozblasSDCopyMat (n, mbk, devA, lda, devATmp, ldat); ozblasSDCopyMat (n, 1, devB, n, devBTmp, ldbt); } else { ozblasSDMallocTmpMat (oh, mbk, n, &devATmp, ldat); ozblasSDMallocTmpMat (oh, mbk, 1, &devBTmp, ldbt); ozblasSDCopyMat (mbk, n, devA, lda, devATmp, ldat); ozblasSDCopyMat (mbk, 1, devB, mbk, devBTmp, ldbt); } #endif if (!memCheck (oh)) break; oh->memAddr = memAddrTmp; mbk = ceil (mbk / 2.); } for (int im = 0; im < ceil((float)m/mbk); im++) { int mbk_ = (m-mbk*im >= mbk) ? mbk : m-mbk*im; // Split of A ----------------------------------- t1 = timer(); oh->numSplitArraysA = numSplitArraysMax; if (tranA == 'N' || tranA == 'n') { #ifdef MKL MKL_OMATCOPY ('c', 't', mbk_, n, fone, devA+im*mbk, lda, devASplit, ldas); // TRANSPOSE #else CBLAS_OMATCOPY (CblasColMajor, CblasTrans, mbk_, n, fone, devA+im*mbk, lda, devASplit, ldas); // TRANSPOSE #endif ozblasSplitMat (oh, 'c', n, mbk_, devASplit, ldas, devASplit, devAmax, ldas, oh->numSplitArraysA, oh->lastFlagA); // TRANSPOSE } else { ozblasSplitMat (oh, 'c', mbk_, n, devA+im*mbk, lda, devASplit, devAmax, ldas, oh->numSplitArraysA, oh->lastFlagA); } oh->t_SplitA += timer() - t1; // Split of B ----------------------------------- t1 = timer(); oh->numSplitArraysB = numSplitArraysMax; if (tranA == 'N' || tranA == 'n') ozblasSplitMat (oh, 'r', n, 1, devB, n, devBSplit, devBmax, ldbs, oh->numSplitArraysB, oh->lastFlagB); else ozblasSplitMat (oh, 'r', mbk, 1, devB, mbk, devBSplit, devBmax, ldbs, oh->numSplitArraysB, oh->lastFlagB); oh->t_SplitB += timer() - t1; // Compute -------------------------------------- t1 = timer(); uint32_t numB, ia, ic = 0; uint32_t maxlevel = (oh->numSplitArraysA-1) + (oh->numSplitArraysB-1); if (oh->fastModeFlag) { if (oh->numSplitArraysMax == 0) maxlevel = MIN (oh->numSplitArraysA-1, oh->numSplitArraysB-1); else maxlevel = oh->numSplitArraysMax-1; } FP_CTYPE *ptrA, *ptrB, *ptrC; ptrC = devCSplit; for (ia = 0; ia < MIN (maxlevel+1, oh->numSplitArraysA); ia++) { if (tranA == 'N' || tranA == 'n') ptrA = devASplit+ldas*mbk_*ia; // TRANSPOSE else ptrA = devASplit+ldas*n*ia; if (oh->useGemmFlag) { numB = MIN (oh->numSplitArraysB, maxlevel+1 - ia); ptrB = devBSplit; if (tranA == 'N' || tranA == 'n') CBLAS_GEMM (CblasColMajor, CblasTrans, CblasNoTrans, mbk_, numB, n, fone, ptrA, ldas, ptrB, ldbs, fzero, ptrC, ldcs); else CBLAS_GEMM (CblasColMajor, CblasTrans, CblasNoTrans, n, numB, mbk_, fone, ptrA, ldas, ptrB, ldbs, fzero, ptrC, ldcs); ptrC += ldcs*numB; ic += numB; } else { for (uint32_t ib = 0; ib < oh->numSplitArraysB; ib++) { if (ia + ib <= maxlevel) { ptrB = devBSplit+ldbs*ib; ptrC = devCSplit+ldcs*ic; CBLAS_GEMV (CblasColMajor, CblasTrans, mbk_, n, fone, ptrA, ldas, ptrB, ione, fzero, ptrC, ione); ic++; } } } } oh->numSplitArraysC = ic; oh->t_comp += timer() - t1; // Sum ----------------------------------------- t1 = timer(); if (tranA == 'N' || tranA == 'n') ozblasSumMat (mbk_, 1, devCSplit, ldcs, ldcs, oh->numSplitArraysC, &devC[im*mbk], ldcs, alpha, beta, oh->sumModeFlag); else ozblasSumMat (n, 1, devCSplit, ldcs, ldcs, oh->numSplitArraysC, &devC[im*mbk], ldcs, alpha, beta, oh->sumModeFlag); oh->t_sum += timer() - t1; } oh->t_total = timer() - t0; #endif return 0; } ozblas-1.2a17/src/cpu/ozblas_axpy.cpp0000644000175200017520000000074513725177602020106 0ustar mukunokimukunoki#include "ozblas_common.h" int OzAXPY ( ozblasHandle_t *oh, const int n, const FP_DTYPE alpha, const FP_DTYPE *devX, const int incx, FP_DTYPE *devY, const int incy ) { if (oh->reproModeFlag == 0) { RCBLAS_AXPY (n, alpha, devX, incx, devY, incy); return 0; } if (incx != 1 || incy != 1) { printf ("incx and incy are not supported.\n"); return 0; } #pragma omp parallel for for (int i = 0; i < n; i++) { devY[i] = fma (alpha, devX[i], devY[i]); } return 0; } ozblas-1.2a17/src/cpu/ozblas_sum_npara.cpp0000644000175200017520000001041513725177602021105 0ustar mukunokimukunoki#include "ozblas_common.h" // ========================================= // NearSum (and AccSum) // from "ACCURATE FLOATING-POINT SUMMATION" by S.M.RUMP, T.OGITA, S.OISHI (2005) // http://www.ti3.tu-harburg.de/paper/rump/RuOgOi06.pdf // ========================================= void ozblasTransformNpara (const int32_t n, FP_CTYPE *vec, const int32_t ld, double rho, double &tau1, double &tau2) { int32_t i, m; double tmp, mu, sigma, t, tau; mu = fabs(vec[0]); for (i = 1; i < n; i++) mu = MAX (mu, fabs(vec[i*ld])); if ((n == 0) || (mu == 0.)) { tau1 = rho; tau2 = 0.; return; } m = ceil (log2((double)(n+2))); sigma = scalbn (1., m+ceil(log2(mu))); t = rho; while (1) { // ExtractVector tau = 0.; #pragma omp parallel for private (tmp) reduction(+:tau) for (i = 0; i < n; i++) { tmp = (sigma + vec[i*ld]) - sigma; vec[i*ld] -= tmp; // <- output tau += tmp; } // here, tau1 = t1 tau1 = t + tau; if ((sigma <= scalbn (1., -1022)) || (fabs(tau1) >= scalbn (1., 2*m+1-53)*sigma)) { //FastTwoSum (t, tau, &tau1, &tau2); //tau1 = t + tau tau2 = tau - (tau1 - t); return; } sigma = scalbn (1., m-53) * sigma; t = tau1; } } void ozblasTransformKNpara (const int32_t n, FP_CTYPE *vec, const int32_t ld, double rho, double &res, double &r) { int32_t i; double tmp, tau1, tau2; ozblasTransformNpara (n, vec, ld, rho, tau1, tau2); tmp = 0.; #pragma omp parallel for reduction(+:tmp) for (i = 0; i < n; i++) { tmp += vec[i*ld]; } res = tau1 + (tau2 + tmp); r = tau2 - (res - tau1); } __inline__ double cuSign (double v) { return (v < 0) ? -1.:1.; } double ozblasNearSumNpara (const int32_t n, FP_CTYPE *vec, const int32_t ld) { double tmp, res, res2, r, r2, mu, delta, delta2; double eps = scalbn (1., -53); ozblasTransformKNpara (n, vec, ld, 0, res, r); ozblasTransformKNpara (n, vec, ld, r, delta, r2); if (delta == 0) return res; res2 = res + cuSign (delta) * eps * fabs(res); if (res2 == res) { mu = cuSign (delta) * eps * fabs(res); res2 = res + 2. * cuSign (delta) * eps * fabs(res); } else { mu = (res2 - res) / 2.; } if (fabs(delta) < fabs(mu)) return res; if (fabs(delta) > fabs(mu)) return res2; ozblasTransformKNpara (n, vec, ld, r2, delta2, tmp); if (delta2 == 0) return res + mu; if (cuSign (delta2) == cuSign (mu)) return res2; return res; } // ========================================= // NearSumMat // ========================================= void ozblasNearSumMatPdevNparaKernel ( const int32_t m, const int32_t n, FP_CTYPE *devCsplit, const int32_t llsc, const int32_t ldsc, const int32_t numsplit, FP_DTYPE *devC, const int32_t ldc, const FP_DTYPE alpha, const FP_DTYPE beta ) { double t; int32_t addrx, addry; for (addrx = 0; addrx < m; addrx++) { for (addry = 0; addry < n; addry++) { t = ozblasNearSumNpara (numsplit, &devCsplit[addry * ldsc + addrx], llsc); if (beta == 0.) devC[addry * ldc + addrx] = (FP_CTYPE)alpha * t; else devC[addry * ldc + addrx] = fma ((FP_CTYPE)alpha, t, (FP_CTYPE)beta * (FP_CTYPE)devC[addry * ldc + addrx]); } } } void ozblasSumMatPdevNparaKernel ( const int32_t m, const int32_t n, const FP_CTYPE *devCsplit, const int32_t llsc, const int32_t ldsc, const int32_t numsplit, FP_DTYPE *devC, const int32_t ldc, const FP_DTYPE alpha, const FP_DTYPE beta ) { FP_CTYPE t; int32_t addrx, addry; for (addrx = 0; addrx < m; addrx++) { for (addry = 0; addry < n; addry++) { t = 0.; // here, do not use OMP (for reproducibility) for (int32_t i = numsplit-1; i >= 0; i--) t += (FP_CTYPE)devCsplit[llsc * i + addry * ldsc + addrx]; if (beta == 0.) devC[addry * ldc + addrx] = (FP_CTYPE)alpha * t; else devC[addry * ldc + addrx] = fma ((FP_CTYPE)alpha, t, (FP_CTYPE)beta * (FP_CTYPE)devC[addry * ldc + addrx]); } } } // parallel for numsplit-wise void ozblasSumMatNpara ( const int32_t m, const int32_t n, FP_CTYPE *devCsplit, const int32_t llsc, const int32_t ldsc, const int32_t numsplit, FP_DTYPE *devC, const int32_t ldc, const FP_DTYPE alpha, const FP_DTYPE beta, const uint32_t sumModeFlag ) { if (sumModeFlag) ozblasNearSumMatPdevNparaKernel (m, n, devCsplit, llsc, ldsc, numsplit, devC, ldc, alpha, beta); else ozblasSumMatPdevNparaKernel (m, n, devCsplit, llsc, ldsc, numsplit, devC, ldc, alpha, beta); } ozblas-1.2a17/src/cpu/ozblas_sum.cpp0000644000175200017520000001015113725177602017721 0ustar mukunokimukunoki#include "ozblas_common.h" // ========================================= // NearSum (and AccSum) // from "ACCURATE FLOATING-POINT SUMMATION" by S.M.RUMP, T.OGITA, S.OISHI (2005) // http://www.ti3.tu-harburg.de/paper/rump/RuOgOi06.pdf // ========================================= void ozblasTransform (const int32_t n, FP_CTYPE *vec, const int32_t ld, double rho, double &tau1, double &tau2) { int32_t i, m; double tmp, mu, sigma, t, tau; mu = fabs(vec[0]); for (i = 1; i < n; i++) mu = MAX (mu, fabs(vec[i*ld])); if ((n == 0) || (mu == 0.)) { tau1 = rho; tau2 = 0.; return; } m = ceil (log2((double)(n+2))); sigma = scalbn (1., m+ceil(log2(mu))); t = rho; while (1) { // ExtractVector tau = 0.; for (i = 0; i < n; i++) { tmp = (sigma + vec[i*ld]) - sigma; vec[i*ld] -= tmp; // <- output tau += tmp; } // here, tau1 = t1 tau1 = t + tau; if ((sigma <= scalbn (1., -1022)) || (fabs(tau1) >= scalbn (1., 2*m+1-53)*sigma)) { //FastTwoSum (t, tau, &tau1, &tau2); //tau1 = t + tau tau2 = tau - (tau1 - t); return; } sigma = scalbn (1., m-53) * sigma; t = tau1; } } void ozblasTransformK (const int32_t n, FP_CTYPE *vec, const int32_t ld, double rho, double &res, double &r) { double tmp, tau1, tau2; ozblasTransform (n, vec, ld, rho, tau1, tau2); tmp = 0.; for (int32_t i = 0; i < n; i++) tmp += vec[i*ld]; res = tau1 + (tau2 + tmp); r = tau2 - (res - tau1); } __inline__ double cuSign (double v) { return (v < 0) ? -1.:1.; } double ozblasNearSum (const int32_t n, FP_CTYPE *vec, const int32_t ld) { double tmp, res, res2, r, r2, mu, delta, delta2; double eps = scalbn (1., -53); ozblasTransformK (n, vec, ld, 0, res, r); ozblasTransformK (n, vec, ld, r, delta, r2); if (delta == 0) return res; res2 = res + cuSign (delta) * eps * fabs(res); if (res2 == res) { mu = cuSign (delta) * eps * fabs(res); res2 = res + 2. * cuSign (delta) * eps * fabs(res); } else { mu = (res2 - res) / 2.; } if (fabs(delta) < fabs(mu)) return res; if (fabs(delta) > fabs(mu)) return res2; ozblasTransformK (n, vec, ld, r2, delta2, tmp); if (delta2 == 0) return res + mu; if (cuSign (delta2) == cuSign (mu)) return res2; return res; } // ========================================= // NearSumMat // ========================================= void ozblasNearSumMatPdevKernel ( const int32_t m, const int32_t n, FP_CTYPE *devCsplit, const int32_t llsc, const int32_t ldsc, const int32_t numsplit, FP_DTYPE *devC, const int32_t ldc, const FP_DTYPE alpha, const FP_DTYPE beta ) { double t; int32_t addrx, addry; #pragma omp parallel for private (addry, t) for (addrx = 0; addrx < m; addrx++) { for (addry = 0; addry < n; addry++) { t = ozblasNearSum (numsplit, &devCsplit[addry * ldsc + addrx], llsc); if (beta == 0.) devC[addry * ldc + addrx] = (FP_CTYPE)alpha * t; else devC[addry * ldc + addrx] = fma ((FP_CTYPE)alpha, t, (FP_CTYPE)beta * (FP_CTYPE)devC[addry * ldc + addrx]); } } } void ozblasSumMatPdevKernel ( const int32_t m, const int32_t n, const FP_CTYPE *devCsplit, const int32_t llsc, const int32_t ldsc, const int32_t numsplit, FP_DTYPE *devC, const int32_t ldc, const FP_DTYPE alpha, const FP_DTYPE beta ) { FP_CTYPE t; int32_t addrx, addry; #pragma omp parallel for private (addry, t) for (addrx = 0; addrx < m; addrx++) { for (addry = 0; addry < n; addry++) { t = 0.; for (int32_t i = numsplit-1; i >= 0; i--) t += (FP_CTYPE)devCsplit[llsc * i + addry * ldsc + addrx]; if (beta == 0.) devC[addry * ldc + addrx] = (FP_CTYPE)alpha * t; else devC[addry * ldc + addrx] = fma ((FP_CTYPE)alpha, t, (FP_CTYPE)beta * (FP_CTYPE)devC[addry * ldc + addrx]); } } } void ozblasSumMat ( const int32_t m, const int32_t n, FP_CTYPE *devCsplit, const int32_t llsc, const int32_t ldsc, const int32_t numsplit, FP_DTYPE *devC, const int32_t ldc, const FP_DTYPE alpha, const FP_DTYPE beta, const uint32_t sumModeFlag ) { if (sumModeFlag) ozblasNearSumMatPdevKernel (m, n, devCsplit, llsc, ldsc, numsplit, devC, ldc, alpha, beta); else ozblasSumMatPdevKernel (m, n, devCsplit, llsc, ldsc, numsplit, devC, ldc, alpha, beta); } ozblas-1.2a17/src/cpu/ozblas_dot.cpp0000644000175200017520000001130713725177602017707 0ustar mukunokimukunoki#include "ozblas_common.h" // ========================================= // InitVec // ========================================= void ozblasInitVecDevice ( const int n, int64_t *hstA ) { #pragma omp parallel for for (int addrx = 0; addrx < n; addrx++) hstA[addrx] = 0; } FP_DTYPE OzDOT ( ozblasHandle_t *oh, const int n, const FP_DTYPE *hstA, const int incx, const FP_DTYPE *hstB, const int incy ) { counterInit (oh); if (oh->reproModeFlag == 0 && oh->numSplitArraysMax == 1) { double t0 = timer(); FP_DTYPE r = RCBLAS_DOT (n, hstA, incx, hstB, incy); oh->t_DOT_NRM2_total += timer() - t0; return r; } if (incx != 1 || incy != 1 ) { fprintf (stderr, "* incx and incy are not supported.\n"); exit (1); } double t1, t0 = timer(); uint32_t numSplitArraysMax = (oh->numSplitArraysMax > 0) ? oh->numSplitArraysMax : NumSplitDefaultMax; int ldas, ldbs, ldcs; FP_CTYPE fone = 1., fzero = 0.; FP_CTYPE *hstAmax, *hstBmax; FP_CTYPE *hstASplit, *hstBSplit, *hstCSplit; if (oh->numSplitArraysMax > 0) numSplitArraysMax = oh->numSplitArraysMax; ozblasMallocSplitVec (oh, n, numSplitArraysMax, &hstASplit, &hstAmax, ldas); ozblasMallocSplitVec (oh, n, numSplitArraysMax, &hstBSplit, &hstBmax, ldbs); ozblasMallocResultVec (oh, 1, numSplitArraysMax, &hstCSplit, ldcs); #if defined (PREC_SD) int ldat, ldbt; FP_CTYPE *hstATmp, *hstBTmp; ozblasSDMallocTmpVec (oh, n, &hstATmp, ldat); ozblasSDMallocTmpVec (oh, n, &hstBTmp, ldbt); ozblasSDCopyMat (n, 1, hstA, n, hstATmp, ldat); ozblasSDCopyMat (n, 1, hstB, n, hstBTmp, ldbt); #endif memCheck (oh); ozblasInitVecDevice (ldcs * numSplitArraysMax * numSplitArraysMax, (int64_t*)hstCSplit); // Split of A ----------------------------------- t1 = timer(); oh->numSplitArraysA = numSplitArraysMax; #if defined (PREC_SD) ozblasSplitMat (oh, 'r', n, 1, hstATmp, n, hstASplit, hstAmax, ldas, oh->numSplitArraysA, oh->lastFlagA); #else ozblasSplitMat (oh, 'r', n, 1, hstA, n, hstASplit, hstAmax, ldas, oh->numSplitArraysA, oh->lastFlagA); #endif oh->t_SplitA = timer() - t1; // Split of B ----------------------------------- t1 = timer(); oh->numSplitArraysB = numSplitArraysMax; #if defined (PREC_SD) ozblasSplitMat (oh, 'r', n, 1, hstBTmp, n, hstBSplit, hstBmax, ldbs, oh->numSplitArraysB, oh->lastFlagB); #else ozblasSplitMat (oh, 'r', n, 1, hstB, n, hstBSplit, hstBmax, ldbs, oh->numSplitArraysB, oh->lastFlagB); #endif oh->t_SplitB = timer() - t1; // Compute -------------------------------------- t1 = timer(); uint32_t ia, ib; uint32_t maxlevel = (oh->numSplitArraysA-1) + (oh->numSplitArraysB-1); if (oh->fastModeFlag) { if (oh->numSplitArraysMax == 0) maxlevel = MIN (oh->numSplitArraysA-1, oh->numSplitArraysB-1); else maxlevel = oh->numSplitArraysMax-1; } if (oh->useGemmFlag) { /* if (oh->useMyGemmFlag) { FP_CTYPE *hstW = (FP_CTYPE*)(oh->hstWork + oh->memAddr); oh->memAddr += sizeof(FP_CTYPE) * oh->numSplitArraysA * oh->numSplitArraysB * SPLIT_VEC_NBX; memCheck (oh); mublasDgemmdot (oh->ch, n, oh->numSplitArraysA, oh->numSplitArraysB, hstASplit, ldas, hstBSplit, ldbs, hstCSplit, oh->numSplitArraysA, hstW); } else { */ CBLAS_GEMM (CblasColMajor, CblasTrans, CblasNoTrans, oh->numSplitArraysA, oh->numSplitArraysB, n, fone, hstASplit, ldas, hstBSplit, ldbs, fzero, hstCSplit, oh->numSplitArraysA); // } oh->numSplitArraysC = oh->numSplitArraysA * oh->numSplitArraysB; // to ensure the consistency with the computation result by GEMM when FS if (oh->fastModeFlag) { for (ia = 0; ia < oh->numSplitArraysA; ia++) { for (ib = 0; ib < oh->numSplitArraysB; ib++) { if (ia + ib > maxlevel) hstCSplit[oh->numSplitArraysA*ib+ia] = 0.; } } } } else { uint32_t ic = 0; FP_CTYPE *ptrA, *ptrB, *ptrC; for (ia = 0; ia < oh->numSplitArraysA; ia++) { for (ib = 0; ib < oh->numSplitArraysB; ib++) { if (ia + ib <= maxlevel) { ptrA = hstASplit+ldas*ia; ptrB = hstBSplit+ldbs*ib; ptrC = hstCSplit+ic; int one = 1; ptrC[0] = CBLAS_DOT (n, ptrA, one, ptrB, one); ic++; } } } oh->numSplitArraysC = ic; } oh->t_comp = timer() - t1; // Sum ----------------------------------------- t1 = timer(); FP_DTYPE ret; ozblasSumMatNpara (1, 1, hstCSplit, 1, 1, oh->numSplitArraysC, &ret, 1, 1., 0., oh->sumModeFlag); //ozblasSumMat (1, 1, hstCSplit, 1, 1, oh->numSplitArraysC, &ret, 1, 1., 0., oh->sumModeFlag); oh->t_sum = timer() - t1; oh->t_total = timer() - t0; // for CG, time // ================================= oh->t_SplitMat_total += 0.; oh->t_SplitVec_total += oh->t_SplitA + oh->t_SplitB; oh->t_Sum_total += oh->t_sum; oh->t_AXPY_SCAL_total += 0.; oh->t_DOT_NRM2_total += oh->t_comp; oh->t_SpMV_SpMM_total += 0.; // ================================= return ret; } ozblas-1.2a17/src/cpu/ozblas_split_sparse_shift.cpp0000644000175200017520000001022213725177602023021 0ustar mukunokimukunoki// ========================================= // ozblasSplitShiftMat // ========================================= void ozblasSplitShiftMatSparseNKernel ( const int32_t m, const FP_CTYPE *devInput, const int32_t *devRowptr, FP_CTYPE *devOutput, FP_CTYPE *devSplit, FP_CTYPE *devMax, int32_t indexbase, const int32_t splitShift ) { FP_CTYPE sigma, max, input, split; #pragma omp parallel for private (max, sigma, input, split) for (int32_t addrx = 0; addrx < m; addrx++) { int32_t dim = devRowptr[addrx+1] - devRowptr[addrx]; int32_t rho = ceil((double)(UNIT+log2((double)dim))/2.); max = devMax[addrx]; sigma = scalbn (1., rho) * NextPowTwo (max) * splitShift; max = 0.; for (int32_t i = devRowptr[addrx]-indexbase; i < devRowptr[addrx+1]-indexbase; i++) { input = devInput[i]; split = (input + sigma) - sigma; input = input - split; devSplit[i] = split; devOutput[i] = input; if (max < fabs(input)) max = fabs(input); } devMax[addrx] = max; } } void ozblasSplitShiftMatSparseEarlyExitNKernel ( const int32_t m, const FP_CTYPE *devInput, const int32_t *devRowptr, FP_CTYPE *devSplit, FP_CTYPE *devMax, int32_t indexbase, const int32_t splitShift ) { FP_CTYPE sigma, max, input, split; #pragma omp parallel for private (max, sigma, input, split) for (int32_t addrx = 0; addrx < m; addrx++) { int32_t dim = devRowptr[addrx+1] - devRowptr[addrx]; int32_t rho = ceil((double)(UNIT+log2((double)dim))/2.); max = fabs(devMax[addrx]); sigma = scalbn (1., rho) * NextPowTwo (max) * splitShift; for (int32_t i = devRowptr[addrx]-indexbase; i < devRowptr[addrx+1]-indexbase; i++) { input = devInput[i]; split = (input + sigma) - sigma; input = input - split; devSplit[i] = split; } } } void ozblasSplitShiftMatSparseDevice ( const char trans, const int32_t m, const FP_CTYPE *devInput, // input matrix (devAwrk) const int32_t *devRowptr, FP_CTYPE *devOutput, // output matrix (devAwrk) FP_CTYPE *devSplit, // split matrices (output): this includes NumSplitMax matrices FP_CTYPE *devMax, uint32_t lastFlag, int32_t indexbase, const int32_t splitShift ) { /* FP_CTYPE *devC; devC = (FP_CTYPE*)malloc (sizeof(FP_CTYPE) * m); */ if (trans == 'n' || trans == 'N') { // PrintMat (m, 1, devMax, m); if (!lastFlag) ozblasSplitShiftMatSparseNKernel (m, devInput, devRowptr, devOutput, devSplit, devMax, indexbase, splitShift); else ozblasSplitShiftMatSparseEarlyExitNKernel (m, devInput, devRowptr, devSplit, devMax, indexbase, splitShift); } else { fprintf (stderr, "error: Split T is not implemented.\n"); exit (1); } /* // for (int32_t i = 0; i < m; i++) // printf ("%d:%a\n", i, devC[i]); free (devC); */ } void ozblasSplitShiftMatSparse ( ozblasHandle_t *oh, const char trans, const int32_t m, const FP_CTYPE *devInput, const int32_t *devRowptr, FP_CTYPE *devSplit, FP_CTYPE *devMax, const int32_t lds, uint32_t &numSplitArrays, uint32_t &lastFlag ) { uint32_t numSplitArraysMax = numSplitArrays; uint32_t i = 1; int32_t one = 1; int32_t indexbase = oh->useGemmFlag; FP_CTYPE check; // Note: indexbase specifies zero-base index (C-style, useGemmFlag=0) or one-base index (F-style, useGemmFlag=1) for MKL // ozblasFindMax^(0) ozblasFindMaxMatSparseDevice (trans, m, devInput, devRowptr, devMax, indexbase); // Split^(0) & ozblasFindMax^(1) lastFlag = (oh->reproModeFlag && numSplitArraysMax == 1) ? 1 : 0; ozblasSplitShiftMatSparseDevice (trans, m, devInput, devRowptr, &devSplit[lds], &devSplit[0], devMax, lastFlag, indexbase, oh->splitShift); while (i < numSplitArraysMax) { // Check if (numSplitArraysMax-1 == i && !oh->reproModeFlag) { i++; break;} // when repromode=0 check = CBLAS_ASUM (m, devMax, one); if (fabs(check) == 0) break; // Split^(i) & ozblasFindMax^(i+1) lastFlag = (oh->reproModeFlag && numSplitArraysMax-1 == i) ? 1 : 0; ozblasSplitShiftMatSparseDevice (trans, m, &devSplit[lds*i], devRowptr, &devSplit[lds*(i+1)], &devSplit[lds*i], devMax, lastFlag, indexbase, oh->splitShift); i++; } numSplitArrays = i; if (lastFlag == 1 && oh->numSplitArraysMax == 0) { fprintf (stderr, "split error: out of memory (the result may be not correct rounding)\n"); exit (1); } } ozblas-1.2a17/src/cpu/ozblas_split_sparse.cpp0000644000175200017520000001167713725177602021643 0ustar mukunokimukunoki#include "ozblas_common.h" // ========================================= // ozblasSplitMat // ========================================= __inline__ double NextPowTwo (const double p) { return scalbn (p, UNIT) - ((scalbn (1., UNIT) - 1) * p); } void ozblasFindMaxMatSparseNKernel ( const int32_t m, const FP_CTYPE *devInput, const int32_t *devRowptr, FP_CTYPE *devMax, int32_t indexbase ) { FP_CTYPE max, input; #pragma omp parallel for private (max, input) for (int32_t addrx = 0; addrx < m; addrx++) { max = 0.; for (int32_t i = devRowptr[addrx]-indexbase; i < devRowptr[addrx+1]-indexbase; i++) { input = devInput[i]; if (max < fabs(input)) max = fabs(input); } devMax[addrx] = max; } } void ozblasFindMaxMatSparseDevice ( const char trans, const int32_t m, const FP_CTYPE *devInput, const int32_t *devRowptr, FP_CTYPE *devMax, int32_t indexbase ) { if (trans == 'n' || trans == 'N') { ozblasFindMaxMatSparseNKernel (m, devInput, devRowptr, devMax, indexbase); } else { fprintf (stderr, "error: Split T is not implemented.\n"); exit (1); } } void ozblasSplitMatSparseNKernel ( const int32_t m, const FP_CTYPE *devInput, const int32_t *devRowptr, FP_CTYPE *devOutput, FP_CTYPE *devSplit, FP_CTYPE *devMax, int32_t indexbase ) { FP_CTYPE sigma, max, input, split; #pragma omp parallel for private (max, sigma, input, split) for (int32_t addrx = 0; addrx < m; addrx++) { int32_t dim = devRowptr[addrx+1] - devRowptr[addrx]; int32_t rho = ceil((double)(UNIT+log2((double)dim))/2.); max = devMax[addrx]; sigma = scalbn (1., rho) * NextPowTwo (max); max = 0.; for (int32_t i = devRowptr[addrx]-indexbase; i < devRowptr[addrx+1]-indexbase; i++) { input = devInput[i]; split = (input + sigma) - sigma; input = input - split; devSplit[i] = split; devOutput[i] = input; if (max < fabs(input)) max = fabs(input); } devMax[addrx] = max; } } void ozblasSplitMatSparseEarlyExitNKernel ( const int32_t m, const FP_CTYPE *devInput, const int32_t *devRowptr, FP_CTYPE *devSplit, FP_CTYPE *devMax, int32_t indexbase ) { FP_CTYPE sigma, max, input, split; #pragma omp parallel for private (max, sigma, input, split) for (int32_t addrx = 0; addrx < m; addrx++) { int32_t dim = devRowptr[addrx+1] - devRowptr[addrx]; int32_t rho = ceil((double)(UNIT+log2((double)dim))/2.); max = fabs(devMax[addrx]); sigma = scalbn (1., rho) * NextPowTwo (max); for (int32_t i = devRowptr[addrx]-indexbase; i < devRowptr[addrx+1]-indexbase; i++) { input = devInput[i]; split = (input + sigma) - sigma; input = input - split; devSplit[i] = split; } } } void ozblasSplitMatSparseDevice ( const char trans, const int32_t m, const FP_CTYPE *devInput, // input matrix (devAwrk) const int32_t *devRowptr, FP_CTYPE *devOutput, // output matrix (devAwrk) FP_CTYPE *devSplit, // split matrices (output): this includes NumSplitMax matrices FP_CTYPE *devMax, uint32_t lastFlag, int32_t indexbase ) { /* FP_CTYPE *devC; devC = (FP_CTYPE*)malloc (sizeof(FP_CTYPE) * m); */ if (trans == 'n' || trans == 'N') { // PrintMat (m, 1, devMax, m); if (!lastFlag) ozblasSplitMatSparseNKernel (m, devInput, devRowptr, devOutput, devSplit, devMax, indexbase); else ozblasSplitMatSparseEarlyExitNKernel (m, devInput, devRowptr, devSplit, devMax, indexbase); } else { fprintf (stderr, "error: Split T is not implemented.\n"); exit (1); } /* // for (int32_t i = 0; i < m; i++) // printf ("%d:%a\n", i, devC[i]); free (devC); */ } void ozblasSplitMatSparse ( ozblasHandle_t *oh, const char trans, const int32_t m, const FP_CTYPE *devInput, const int32_t *devRowptr, FP_CTYPE *devSplit, FP_CTYPE *devMax, const int32_t lds, uint32_t &numSplitArrays, uint32_t &lastFlag ) { uint32_t numSplitArraysMax = numSplitArrays; uint32_t i = 1; int32_t one = 1; int32_t indexbase = oh->useGemmFlag; FP_CTYPE check; // Note: indexbase specifies zero-base index (C-style, useGemmFlag=0) or one-base index (F-style, useGemmFlag=1) for MKL // ozblasFindMax^(0) ozblasFindMaxMatSparseDevice (trans, m, devInput, devRowptr, devMax, indexbase); // Split^(0) & ozblasFindMax^(1) lastFlag = (oh->reproModeFlag && numSplitArraysMax == 1) ? 1 : 0; ozblasSplitMatSparseDevice (trans, m, devInput, devRowptr, &devSplit[lds], &devSplit[0], devMax, lastFlag, indexbase); while (i < numSplitArraysMax) { // Check if (numSplitArraysMax-1 == i && !oh->reproModeFlag) { i++; break;} // when repromode=0 check = CBLAS_ASUM (m, devMax, one); if (fabs(check) == 0) break; // Split^(i) & ozblasFindMax^(i+1) lastFlag = (oh->reproModeFlag && numSplitArraysMax-1 == i) ? 1 : 0; ozblasSplitMatSparseDevice (trans, m, &devSplit[lds*i], devRowptr, &devSplit[lds*(i+1)], &devSplit[lds*i], devMax, lastFlag, indexbase); i++; } numSplitArrays = i; if (lastFlag == 1 && oh->numSplitArraysMax == 0) { fprintf (stderr, "split error: out of memory (the result may be not correct rounding)\n"); exit (1); } } #include "ozblas_split_sparse_shift.cpp" ozblas-1.2a17/src/cpu/ozblas_split.cpp0000644000175200017520000002006713725177602020257 0ustar mukunokimukunoki#include "ozblas_common.h" __inline__ double NextPowTwo (const double p) { return scalbn (p, UNIT) - ((scalbn (1., UNIT) - 1) * p); } // ========================================= // Split // Based on K.Ozaki et al., "Error-free transformations of matrix multiplication // by using fast routines of matrix multiplication and its applications", 2012 // ========================================= void ozblasSplitVecKernel ( const int32_t n, const int32_t rho, const FP_CTYPE *devInput, FP_CTYPE *devOutput, FP_CTYPE *devSplit, FP_CTYPE *devMax ) { FP_CTYPE sigma, max_, input, split; max_ = devMax[0]; sigma = scalbn (1., rho) * NextPowTwo (max_); max_ = 0.; #pragma omp parallel for private (input, split) reduction(max:max_) for (int32_t i = 0; i < n; i++) { input = devInput[i]; split = (input + sigma) - sigma; input = input - split; devSplit[i] = split; devOutput[i] = input; if (max_ < fabs(input)) { max_ = fabs(input); } } devMax[0] = max_; } void ozblasSplitVecEarlyExitKernel ( const int32_t n, const int32_t rho, const FP_CTYPE *devInput, FP_CTYPE *devSplit, FP_CTYPE *devMax ) { FP_CTYPE sigma, max, input, split; max = devMax[0]; sigma = scalbn (1., rho) * NextPowTwo (max); #pragma omp parallel for private (input, split) for (int32_t i = 0; i < n; i++) { input = devInput[i]; split = (input + sigma) - sigma; input = input - split; devSplit[i] = split; } } // ========================================= // ozblasSplitMat // ========================================= void ozblasFindMaxMatNKernel ( const int32_t m, const int32_t n, const FP_CTYPE *devInput, const int32_t ldi, FP_CTYPE *devMax ) { FP_CTYPE max, input; #pragma omp parallel for private (max, input) for (int32_t addrx = 0; addrx < m; addrx++) { max = 0.; for (int32_t j = 0; j < n; j++) { input = devInput[j * ldi + addrx]; if (max < fabs(input)) max = fabs(input); } devMax[addrx] = max; } } void ozblasFindMaxMatTKernel ( const int32_t m, const int32_t n, const FP_CTYPE *devInput, const int32_t ldi, FP_CTYPE *devMax ) { FP_CTYPE max, input; #pragma omp parallel for private (max, input) for (int32_t addry = 0; addry < n; addry++) { max = 0.; for (int32_t i = 0; i < m; i++) { input = devInput[addry * ldi + i]; if (max < fabs(input)) max = fabs(input); } devMax[addry] = max; } } void ozblasFindMaxMatDevice ( const char major, const int32_t m, const int32_t n, const FP_CTYPE *devInput, const int32_t ldi, FP_CTYPE *devMax ) { if (major == 'r') ozblasFindMaxMatNKernel (m, n, devInput, ldi, devMax); else ozblasFindMaxMatTKernel (m, n, devInput, ldi, devMax); } void ozblasSplitMatNKernel ( const int32_t m, const int32_t n, const int32_t rho, const FP_CTYPE *devInput, const int32_t ldi, FP_CTYPE *devOutput, FP_CTYPE *devSplit, const int32_t lds, FP_CTYPE *devMax ) { FP_CTYPE sigma, max, input, split; #pragma omp parallel for private (max, sigma, input, split) for (int32_t addrx = 0; addrx < m; addrx++) { max = devMax[addrx]; sigma = scalbn (1., rho) * NextPowTwo (max); max = 0.; for (int32_t j = 0; j < n; j++) { input = devInput[j * ldi + addrx]; split = (input + sigma) - sigma; input = input - split; devSplit[j * lds + addrx] = split; devOutput[j * lds + addrx] = input; if (max < fabs(input)) max = fabs(input); } devMax[addrx] = max; } } void ozblasSplitMatTKernel ( const int32_t m, const int32_t n, const int32_t rho, const FP_CTYPE *devInput, const int32_t ldi, FP_CTYPE *devOutput, FP_CTYPE *devSplit, const int32_t lds, FP_CTYPE *devMax ) { FP_CTYPE sigma, max, input, split; #pragma omp parallel for private (max, sigma, input, split) for (int32_t addry = 0; addry < n; addry++) { max = devMax[addry]; sigma = scalbn (1., rho) * NextPowTwo (max); max = 0.; for (int32_t i = 0; i < m; i++) { input = devInput[addry * ldi + i]; split = (input + sigma) - sigma; input = input - split; devSplit[addry * lds + i] = split; devOutput[addry * lds + i] = input; if (max < fabs(input)) max = fabs(input); } devMax[addry] = max; } } void ozblasSplitMatEarlyExitNKernel ( const int32_t m, const int32_t n, const int32_t rho, const FP_CTYPE *devInput, const int32_t ldi, FP_CTYPE *devSplit, const int32_t lds, FP_CTYPE *devMax ) { FP_CTYPE sigma, max, input, split; #pragma omp parallel for private (max, sigma, input, split) for (int32_t addrx = 0; addrx < m; addrx++) { max = devMax[addrx]; sigma = scalbn (1., rho) * NextPowTwo (max); for (int32_t j = 0; j < n; j++) { input = devInput[j * ldi + addrx]; split = (input + sigma) - sigma; input = input - split; devSplit[j * lds + addrx] = split; } } } void ozblasSplitMatEarlyExitTKernel ( const int32_t m, const int32_t n, const int32_t rho, const FP_CTYPE *devInput, const int32_t ldi, FP_CTYPE *devSplit, const int32_t lds, FP_CTYPE *devMax ) { FP_CTYPE sigma, max, input, split; #pragma omp parallel for private (max, sigma, input, split) for (int32_t addry = 0; addry < n; addry++) { max = devMax[addry]; sigma = scalbn (1., rho) * NextPowTwo (max); for (int32_t i = 0; i < m; i++) { input = devInput[addry * ldi + i]; split = (input + sigma) - sigma; input = input - split; devSplit[addry * lds + i] = split; } } } void ozblasSplitMatDevice ( const char major, const int32_t m, const int32_t n, const FP_CTYPE *devInput, // input matrix (devAwrk) const int32_t ldi, // leading dimension of input matrix FP_CTYPE *devOutput, // output matrix (devAwrk) FP_CTYPE *devSplit, // split matrices (output): this includes NumSplitMax matrices const int32_t lds, // leading dimension of split matrix (# of cols) FP_CTYPE *devMax, uint32_t lastFlag ) { const int32_t dim = ((major == 'r') && n > 1) ? n : m; int32_t rho = ceil((double)(UNIT+log2((double)dim))/2.); if (major == 'r') { if (n == 1) { if (!lastFlag) ozblasSplitVecKernel (m, rho, devInput, devOutput, devSplit, devMax); else ozblasSplitVecEarlyExitKernel (m, rho, devInput, devSplit, devMax); } else { if (!lastFlag) ozblasSplitMatNKernel (m, n, rho, devInput, ldi, devOutput, devSplit, lds, devMax); else ozblasSplitMatEarlyExitNKernel (m, n, rho, devInput, ldi, devSplit, lds, devMax); } } else { if (!lastFlag) ozblasSplitMatTKernel (m, n, rho, devInput, ldi, devOutput, devSplit, lds, devMax); else ozblasSplitMatEarlyExitTKernel (m, n, rho, devInput, ldi, devSplit, lds, devMax); } } void ozblasSplitMat ( ozblasHandle_t *oh, const char major, const int32_t m, const int32_t n, const FP_CTYPE *devInput, const int32_t ld, FP_CTYPE *devSplit, FP_CTYPE *devMax, const int32_t lds, uint32_t &numSplitArrays, uint32_t &lastFlag ) { uint32_t numSplitArraysMax = numSplitArrays; uint32_t i = 1; int32_t one = 1; FP_CTYPE check; // FindMax^(0) if (n == 1 && major == 'r') { int32_t ptrMax = 0; ptrMax = CBLAS_IAMAX (m, devInput, one); // Warning: note that cublas returns the index on the basis of 1-origin, while MKL (CBLAS) returns it on the basis of 0-origin !! devMax[0] = devInput[ptrMax]; // so, here do not -1 !! } else { ozblasFindMaxMatDevice (major, m, n, devInput, ld, devMax); } // Split^(0) & FindMax^(1) lastFlag = (oh->reproModeFlag && numSplitArraysMax == 1) ? 1 : 0; // in, out, split ozblasSplitMatDevice (major, m, n, devInput, ld, &devSplit[lds*n], &devSplit[0], lds, devMax, lastFlag); while (i < numSplitArraysMax) { // Check if (numSplitArraysMax-1 == i && !oh->reproModeFlag) { i++; break;} // when repromode=0 if (n == 1 && major == 'r') check = devMax[0]; else check = CBLAS_ASUM ((major=='r')?m:n, devMax, one); if (fabs(check) == 0) break; // Split^(i) & FindMax^(i+1) lastFlag = (oh->reproModeFlag && numSplitArraysMax-1 == i) ? 1 : 0; ozblasSplitMatDevice (major, m, n, &devSplit[lds*n*i], lds, &devSplit[lds*n*(i+1)], &devSplit[lds*n*i], lds, devMax, lastFlag); i++; } numSplitArrays = i; if (lastFlag == 1 && oh->numSplitArraysMax == 0) { fprintf (stderr, "* split error: out of memory (the result may be not correct rounding)\n"); exit (1); } } #include "ozblas_split_shift.cpp" ozblas-1.2a17/src/cpu/ozblas.cpp0000644000175200017520000000631713725177602017046 0ustar mukunokimukunoki#include "ozblas_common.h" double timer () { struct timeval tv; gettimeofday (&tv, NULL); return tv.tv_sec + (double) tv.tv_usec * 1.0e-6; } int32_t getPitchSize (int32_t n) { return ceil((double)n / 128) * 128; } uint32_t memCheck (ozblasHandle_t *oh) { if (oh->memAddr > oh->workSizeBytes) return 1; return 0; } void counterInit (ozblasHandle_t *oh) { oh->t_SplitA = 0.; oh->t_SplitB = 0.; oh->t_comp = 0.; oh->t_sum = 0.; oh->t_total = 0.; oh->numSplitArraysA = 0; oh->numSplitArraysB = 0; oh->numSplitArraysC = 0; oh->lastFlagA = 0; oh->lastFlagB = 0; oh->memAddr = oh->memMaskSplitA; } void ozblasCreate (ozblasHandle_t *oh, uint64_t WorkSizeBytes, uint32_t NumSplitArraysMax, uint32_t FastMode, uint32_t ReproMode, uint32_t SumMode) { oh->workSizeBytes = (WorkSizeBytes) ? WorkSizeBytes : DefaultWorkSize; oh->numSplitArraysMax = NumSplitArraysMax; oh->fastModeFlag = FastMode; oh->reproModeFlag = ReproMode; oh->sumModeFlag = SumMode; // for CG oh->memMaskSplitA = 0; // disable pre-split of matA oh->splitShift = 1; // default (no-Splitshift) // experimental parameters (default) oh->useGemmFlag = 1; oh->useMyGemmFlag = 1; #ifdef MKL oh->useBatchedGemmFlag = 1; #else oh->useBatchedGemmFlag = 0; // currently OpenBLAS does not support it #endif // oh->sparseThreshold = 0.; // work memory allocation char *devWork_; devWork_ = (char*) malloc (oh->workSizeBytes); oh->devWork = devWork_; // batched BLAS if (oh->useBatchedGemmFlag) { char *devBatchAddr_; devBatchAddr_ = (char*) malloc (sizeof(double*) * NumSplitDefaultMax * NumSplitDefaultMax * 3); oh->devBatchAddr = devBatchAddr_; } } void ozblasProfilingSetting (ozblasHandle_t *oh, uint32_t useGemmFlag, uint32_t useMyGemmFlag, uint32_t useBatchedGemmFlag) { oh->useGemmFlag = useGemmFlag; oh->useMyGemmFlag = useMyGemmFlag; #ifdef MKL oh->useBatchedGemmFlag = useBatchedGemmFlag; #else oh->useBatchedGemmFlag = 0; #endif } void ozblasSetNumSplitArraysMax (ozblasHandle_t *oh, uint32_t NumSplitArraysMax) { oh->numSplitArraysMax = NumSplitArraysMax; } void ozblasSetFastMode (ozblasHandle_t *oh, uint32_t FastMode) { oh->fastModeFlag = (FastMode == 0) ? 0 : 1; } void ozblasSetReproMode (ozblasHandle_t *oh, uint32_t ReproMode) { oh->reproModeFlag = (ReproMode == 0) ? 0 : 1; } void ozblasSetSumMode (ozblasHandle_t *oh, uint32_t SumMode) { oh->sumModeFlag = (SumMode == 0) ? 0 : 1; } uint32_t ozblasGetNumSplitArraysMax (ozblasHandle_t *oh) { return oh->numSplitArraysMax; } uint32_t ozblasGetFastMode (ozblasHandle_t *oh) { return oh->fastModeFlag; } uint32_t ozblasGetReproMode (ozblasHandle_t *oh) { return oh->reproModeFlag; } uint32_t ozblasGetSumMode (ozblasHandle_t *oh) { return oh->sumModeFlag; } void ozblasDestroy (ozblasHandle_t *oh) { free (oh->devWork); if (oh->useBatchedGemmFlag) free (oh->devBatchAddr); } CBLAS_TRANSPOSE ToCblasOp (char tran) { if (tran == 'N' || tran == 'n') return CblasNoTrans; if (tran == 'T' || tran == 't') return CblasTrans; if (tran == 'C' || tran == 'c') return CblasConjTrans; return CblasNoTrans; } char FromCublasOp (CBLAS_TRANSPOSE tran) { if (tran == CblasNoTrans) return 'n'; if (tran == CblasTrans) return 't'; if (tran == CblasConjTrans) return 'c'; return 'n'; } ozblas-1.2a17/src/cpu/ozblas_split_shift.cpp0000644000175200017520000001634413725177602021457 0ustar mukunokimukunoki// ========================================= // Split // Based on K.Ozaki et al., "Error-free transformations of matrix multiplication // by using fast routines of matrix multiplication and its applications", 2012 // ========================================= void ozblasSplitShiftVecKernel ( const int32_t n, const int32_t rho, const FP_CTYPE *devInput, FP_CTYPE *devOutput, FP_CTYPE *devSplit, FP_CTYPE *devMax, const int32_t splitShift ) { FP_CTYPE sigma, max_, input, split; max_ = devMax[0]; sigma = scalbn (1., rho) * NextPowTwo (max_) / splitShift; max_ = 0.; #pragma omp parallel for private (input, split) reduction(max:max_) for (int32_t i = 0; i < n; i++) { input = devInput[i]; split = (input + sigma) - sigma; input = input - split; devSplit[i] = split; devOutput[i] = input; if (max_ < fabs(input)) { max_ = fabs(input); } } devMax[0] = max_; } void ozblasSplitShiftVecEarlyExitKernel ( const int32_t n, const int32_t rho, const FP_CTYPE *devInput, FP_CTYPE *devSplit, FP_CTYPE *devMax, const int32_t splitShift ) { FP_CTYPE sigma, max, input, split; max = devMax[0]; sigma = scalbn (1., rho) * NextPowTwo (max) / splitShift; #pragma omp parallel for private (input, split) for (int32_t i = 0; i < n; i++) { input = devInput[i]; split = (input + sigma) - sigma; input = input - split; devSplit[i] = split; } } // ========================================= // ozblasSplitShiftMat // ========================================= void ozblasSplitShiftMatNKernel ( const int32_t m, const int32_t n, const int32_t rho, const FP_CTYPE *devInput, const int32_t ldi, FP_CTYPE *devOutput, FP_CTYPE *devSplit, const int32_t lds, FP_CTYPE *devMax, const int32_t splitShift ) { FP_CTYPE sigma, max, input, split; #pragma omp parallel for private (max, sigma, input, split) for (int32_t addrx = 0; addrx < m; addrx++) { max = devMax[addrx]; sigma = scalbn (1., rho) * NextPowTwo (max) / splitShift; max = 0.; for (int32_t j = 0; j < n; j++) { input = devInput[j * ldi + addrx]; split = (input + sigma) - sigma; input = input - split; devSplit[j * lds + addrx] = split; devOutput[j * lds + addrx] = input; if (max < fabs(input)) max = fabs(input); } devMax[addrx] = max; } } void ozblasSplitShiftMatTKernel ( const int32_t m, const int32_t n, const int32_t rho, const FP_CTYPE *devInput, const int32_t ldi, FP_CTYPE *devOutput, FP_CTYPE *devSplit, const int32_t lds, FP_CTYPE *devMax, const int32_t splitShift ) { FP_CTYPE sigma, max, input, split; #pragma omp parallel for private (max, sigma, input, split) for (int32_t addry = 0; addry < n; addry++) { max = devMax[addry]; sigma = scalbn (1., rho) * NextPowTwo (max) / splitShift; max = 0.; for (int32_t i = 0; i < m; i++) { input = devInput[addry * ldi + i]; split = (input + sigma) - sigma; input = input - split; devSplit[addry * lds + i] = split; devOutput[addry * lds + i] = input; if (max < fabs(input)) max = fabs(input); } devMax[addry] = max; } } void ozblasSplitShiftMatEarlyExitNKernel ( const int32_t m, const int32_t n, const int32_t rho, const FP_CTYPE *devInput, const int32_t ldi, FP_CTYPE *devSplit, const int32_t lds, FP_CTYPE *devMax, const int32_t splitShift ) { FP_CTYPE sigma, max, input, split; #pragma omp parallel for private (max, sigma, input, split) for (int32_t addrx = 0; addrx < m; addrx++) { max = devMax[addrx]; sigma = scalbn (1., rho) * NextPowTwo (max) / splitShift; for (int32_t j = 0; j < n; j++) { input = devInput[j * ldi + addrx]; split = (input + sigma) - sigma; input = input - split; devSplit[j * lds + addrx] = split; } } } void ozblasSplitShiftMatEarlyExitTKernel ( const int32_t m, const int32_t n, const int32_t rho, const FP_CTYPE *devInput, const int32_t ldi, FP_CTYPE *devSplit, const int32_t lds, FP_CTYPE *devMax, const int32_t splitShift ) { FP_CTYPE sigma, max, input, split; #pragma omp parallel for private (max, sigma, input, split) for (int32_t addry = 0; addry < n; addry++) { max = devMax[addry]; sigma = scalbn (1., rho) * NextPowTwo (max) / splitShift; for (int32_t i = 0; i < m; i++) { input = devInput[addry * ldi + i]; split = (input + sigma) - sigma; input = input - split; devSplit[addry * lds + i] = split; } } } void ozblasSplitShiftMatDevice ( const char major, const int32_t m, const int32_t n, const FP_CTYPE *devInput, // input matrix (devAwrk) const int32_t ldi, // leading dimension of input matrix FP_CTYPE *devOutput, // output matrix (devAwrk) FP_CTYPE *devSplit, // split matrices (output): this includes NumSplitMax matrices const int32_t lds, // leading dimension of split matrix (# of cols) FP_CTYPE *devMax, uint32_t lastFlag, const int32_t splitShift ) { const int32_t dim = ((major == 'r') && n > 1) ? n : m; int32_t rho = ceil((double)(UNIT+log2((double)dim))/2.); if (major == 'r') { if (n == 1) { if (!lastFlag) ozblasSplitShiftVecKernel (m, rho, devInput, devOutput, devSplit, devMax, splitShift); else ozblasSplitShiftVecEarlyExitKernel (m, rho, devInput, devSplit, devMax, splitShift); } else { if (!lastFlag) ozblasSplitShiftMatNKernel (m, n, rho, devInput, ldi, devOutput, devSplit, lds, devMax, splitShift); else ozblasSplitShiftMatEarlyExitNKernel (m, n, rho, devInput, ldi, devSplit, lds, devMax, splitShift); } } else { if (!lastFlag) ozblasSplitShiftMatTKernel (m, n, rho, devInput, ldi, devOutput, devSplit, lds, devMax, splitShift); else ozblasSplitShiftMatEarlyExitTKernel (m, n, rho, devInput, ldi, devSplit, lds, devMax, splitShift); } } void ozblasSplitShiftMat ( ozblasHandle_t *oh, const char major, const int32_t m, const int32_t n, const FP_CTYPE *devInput, const int32_t ld, FP_CTYPE *devSplit, FP_CTYPE *devMax, const int32_t lds, uint32_t &numSplitArrays, uint32_t &lastFlag ) { uint32_t numSplitArraysMax = numSplitArrays; uint32_t i = 1; int32_t one = 1; FP_CTYPE check; // FindMax^(0) if (n == 1 && major == 'r') { int32_t ptrMax = 0; ptrMax = CBLAS_IAMAX (m, devInput, one); // Warning: note that cublas returns the index on the basis of 1-origin, while MKL (CBLAS) returns it on the basis of 0-origin !! devMax[0] = devInput[ptrMax]; // so, here do not -1 !! } else { ozblasFindMaxMatDevice (major, m, n, devInput, ld, devMax); } // Split^(0) & FindMax^(1) lastFlag = (oh->reproModeFlag && numSplitArraysMax == 1) ? 1 : 0; // in, out, split ozblasSplitShiftMatDevice (major, m, n, devInput, ld, &devSplit[lds*n], &devSplit[0], lds, devMax, lastFlag, oh->splitShift); while (i < numSplitArraysMax) { // Check if (numSplitArraysMax-1 == i && !oh->reproModeFlag) { i++; break;} // when repromode=0 if (n == 1 && major == 'r') check = devMax[0]; else check = CBLAS_ASUM ((major=='r')?m:n, devMax, one); if (fabs(check) == 0) break; // Split^(i) & FindMax^(i+1) lastFlag = (oh->reproModeFlag && numSplitArraysMax-1 == i) ? 1 : 0; ozblasSplitShiftMatDevice (major, m, n, &devSplit[lds*n*i], lds, &devSplit[lds*n*(i+1)], &devSplit[lds*n*i], lds, devMax, lastFlag, oh->splitShift); i++; } numSplitArrays = i; if (lastFlag == 1 && oh->numSplitArraysMax == 0) { fprintf (stderr, "* split error: out of memory (the result may be not correct rounding)\n"); exit (1); } } ozblas-1.2a17/src/cpu/Makefile0000644000175200017520000000354113725177602016504 0ustar mukunokimukunoki# Makefile include ../../make.inc AR = ar LIB := ../../lib/libozblas.a FLAGS := -Wall -MD ifeq ($(CXX),icpc) FLAGS = -O3 -fp-model source -fprotect-parens -qopenmp ifeq ($(CPU_ARCH),KNL) FLAGS += -xMIC-AVX512 endif ifeq ($(CPU_ARCH),SKYLAKE) FLAGS += -xCORE-AVX2 -mtune=skylake-avx512 endif # Warning 1: fprotect-parens MUST be used (g++ enables it in default) # Warning 2: fp-model source MUST be used (disable 80-bit register) else FLAGS += -fopenmp endif ifeq ($(CPU_BLAS),MKL) FLAGS += -DMKL endif ifeq ($(CPU_BLAS),OpenBLAS) FLAGS += -I${CBLAS_INC} endif ifeq ($(NVBLAS),yes) FLAGS += -I/usr/local/cuda/include -DNVBLAS endif all: blas ALLSRC += \ ozblas_auxiliary.cpp \ ozblas_split.cpp \ ozblas_sum.cpp \ ozblas_sum_npara.cpp ALLHDR = \ ozblas_common.h \ ozblas_split_shift.cpp \ ozblas_internal.h ifeq ($(AXPY),yes) ALLSRC += ozblas_axpy.cpp endif ifeq ($(DOT),yes) ALLSRC += ozblas_dot.cpp endif ifeq ($(NRM2),yes) ALLSRC += ozblas_nrm2.cpp endif ifeq ($(GEMV),yes) ALLSRC += ozblas_gemv.cpp endif ifeq ($(GEMV),yes) ALLSRC += ozblas_gemv.cpp endif ifeq ($(GEMM),yes) ALLSRC += ozblas_gemm.cpp endif ifeq ($(CSRMV),yes) ifeq ($(CPU_BLAS),MKL) ALLSRC += ozblas_csrmv.cpp \ ozblas_split_sparse.cpp ALLHDR += ozblas_split_sparse_shift.cpp endif ifeq ($(CG),yes) ALLSRC += ozblas_cg.cpp endif endif ALLOBJ = ozblas.o ifeq ($(PREC_S),yes) ALLOBJ += $(ALLSRC:.cpp=.s.o) endif ifeq ($(PREC_D),yes) ALLOBJ += $(ALLSRC:.cpp=.d.o) endif ifeq ($(PREC_SD),yes) ALLOBJ += $(ALLSRC:.cpp=.sd.o) endif $(ALLOBJ) : $(ALLHDR) blas: $(ALLOBJ) $(AR) rv $(LIB) $(ALLOBJ) #%.o: %.cpp # $(CXX) $(FLAGS) -o $@ -c $< %.s.o: %.cpp $(CXX) $(FLAGS) -o $@ -c $< -DPREC_S %.d.o: %.cpp $(CXX) $(FLAGS) -o $@ -c $< -DPREC_D %.sd.o: %.cpp $(CXX) $(FLAGS) -o $@ -c $< -DPREC_SD ozblas.o: ozblas.cpp $(CXX) $(FLAGS) -o $@ -c $< clean : $(RM) *.o *.d $(RM) $(LIB) ozblas-1.2a17/src/cpu/ozblas_cg.cpp0000644000175200017520000001567113725200211017501 0ustar mukunokimukunoki#include "ozblas_common.h" void ozblasCopyVec ( const size_t n, const FP_DTYPE *devIn, FP_DTYPE *devOut ) { #pragma omp parallel for for (size_t i = 0; i < n; i++) { devOut[i] = devIn[i]; } } // true residual is computed using standard FP operations FP_DTYPE getTrueResidual ( ozblasHandle_t *oh, const char tranA, const int32_t dimN, const int32_t dimNNZ, const char *descrA, const FP_DTYPE *matA, const int32_t *matArowptr, const int32_t *matAcolind, const FP_DTYPE *vecB, const FP_DTYPE *vecX, FP_DTYPE *vecT ) { FP_DTYPE res0, fone = 1., fnegone = -1.; // residual: r_i = b-Ax_i ozblasCopyVec (dimN, vecB, vecT); // t = b RMKL_CSRMV (&tranA, &dimN, &dimN, &fnegone, descrA, matA, matAcolind, matArowptr, matArowptr+1, vecX, &fone, vecT); // t = t-Ax (b-Ax) res0 = RCBLAS_NRM2 (dimN, vecT, 1); return res0; } int32_t OzCG ( ozblasHandle_t *oh, const char tranA, const int32_t dimN, const int32_t dimNNZ, const char *descrA, const FP_DTYPE *matA, const int32_t *matArowptr, const int32_t *matAcolind, const FP_DTYPE *vecB, FP_DTYPE *vecX, int32_t maxiter, double tol ) { // ================================= oh->t_SplitMat_total = 0.; oh->t_SplitVec_total = 0.; oh->t_Sum_total = 0.; oh->t_AXPY_SCAL_total = 0.; oh->t_DOT_NRM2_total = 0.; oh->t_SpMV_SpMM_total = 0.; // ================================= int32_t numiter; double t0, t1, t2, t3; float *dat1, *dat2; if (oh->verbose > 0) { dat1 = (float *)malloc(sizeof(float)*maxiter/oh->verbose+1); dat2 = (float *)malloc(sizeof(float)*maxiter/oh->verbose+1); } t1 = timer(); // ----------------------------------------------------- FP_DTYPE alpha, beta, negalpha, resi, res0, nrmb, tmp, dnew, dold; FP_DTYPE fzero = 0., fone = 1., fnegone = -1.; FP_DTYPE *vecT, *vecR, *vecP, *vecQ; vecT = (FP_DTYPE *)malloc(sizeof(FP_DTYPE) * dimN); vecR = (FP_DTYPE *)malloc(sizeof(FP_DTYPE) * dimN); vecP = (FP_DTYPE *)malloc(sizeof(FP_DTYPE) * dimN); vecQ = (FP_DTYPE *)malloc(sizeof(FP_DTYPE) * dimN); #if defined (PREC_SD) printf ("## * On PREC_SD mat-split is performed at each iterration\n"); float *matASplit; matASplit = (float*)matA; #else FP_DTYPE *matASplit; OzCSRMVSplitA (oh, tranA, dimN, dimN, dimNNZ, descrA, matA, matArowptr, matAcolind, &matASplit); #endif nrmb = OzNRM2 (oh, dimN, vecB, 1); // nrmb = |b| // residual: r_0 = b-Ax_0 ozblasCopyVec (dimN, vecB, vecR); // r = b OzCSRMV (oh, tranA, dimN, dimN, dimNNZ, fnegone, descrA, matASplit, matArowptr, matAcolind, vecX, fone, vecR); // r = r-Ax (b-Ax) dold = OzDOT (oh, dimN, vecR, 1, vecR, 1); // dold = resi = sqrt (dold); // resi = |r| if (oh->verbose > 0) { t3 = timer(); dat1[0] = resi/nrmb; dat2[0] = (oh->trueres==0) ? t3-t1 : getTrueResidual (oh, tranA, dimN, dimNNZ, descrA, matA, matArowptr, matAcolind, vecB, vecX, vecT) / nrmb; } ozblasCopyVec (dimN, vecR, vecP); // p = r numiter = 0; t2 = timer(); // ----------------------------------------------------- while (numiter < maxiter) { numiter++; OzCSRMV (oh, tranA, dimN, dimN, dimNNZ, fone, descrA, matASplit, matArowptr, matAcolind, vecP, fzero, vecQ); // q = Ap tmp = OzDOT (oh, dimN, vecP, 1, vecQ, 1); // tmp = alpha = dold / tmp; negalpha = -alpha; t0 = timer(); OzAXPY (oh, dimN, alpha, vecP, 1, vecX, 1); // x = x+alpha*p OzAXPY (oh, dimN, negalpha, vecQ, 1, vecR, 1); // r = r-alpha*q oh->t_AXPY_SCAL_total += timer() - t0; dnew = OzDOT (oh, dimN, vecR, 1, vecR, 1); // dnew = resi = sqrt(dnew); beta = dnew / dold; // beta = dnew/dold dold = dnew; t0 = timer(); CBLAS_SCAL (dimN, beta, vecP, 1); // p = beta*p OzAXPY (oh, dimN, fone, vecR, 1, vecP, 1); // p = p+r oh->t_AXPY_SCAL_total += timer() - t0; if (oh->verbose > 0) { if (numiter%oh->verbose == 0 ) { t3 = timer(); dat1[numiter/oh->verbose] = resi/nrmb; dat2[numiter/oh->verbose] = (oh->trueres==0) ? t3-t1 : getTrueResidual (oh, tranA, dimN, dimNNZ, descrA, matA, matArowptr, matAcolind, vecB, vecX, vecT) / nrmb; } } if (resi/nrmb < tol) break; } t3 = timer(); // ----------------------------------------------------- // ================================= double tloc_SpMV_SpMM_total = oh->t_SpMV_SpMM_total; double tloc_DOT_NRM2_total = oh->t_DOT_NRM2_total; double tloc_AXPY_SCAL_total = oh->t_AXPY_SCAL_total; double tloc_Sum_total = oh->t_Sum_total; double tloc_SplitVec_total = oh->t_SplitVec_total; double tloc_SplitMat_total = oh->t_SplitMat_total; double tloc_Other_total = (t3-t1) -tloc_SpMV_SpMM_total -tloc_DOT_NRM2_total -tloc_AXPY_SCAL_total -tloc_Sum_total -tloc_SplitVec_total -tloc_SplitMat_total; // ================================= if (oh->verbose > 0) { // residual: r_i = b-Ax_i (correctly-rounded) ozblasCopyVec (dimN, vecB, vecT); // t = b oh->memMaskSplitA = 0; // this is needed oh->numSplitArraysMax = 0; oh->reproModeFlag = 1; oh->fastModeFlag = 0; oh->sumModeFlag = 1; OzCSRMV (oh, tranA, dimN, dimN, dimNNZ, fnegone, descrA, matA, matArowptr, matAcolind, vecX, fone, vecT); // t = t-Ax (b-Ax) res0 = OzNRM2 (oh, dimN, vecT, 1); // note: parameter-reset is needed here if user wants to continue the use of OzBLAS later on if (oh->trueres) printf ("\n## iter\t||r_i||/||b||\t||b-Ax||/||b||\n"); else printf ("\n## iter\t||r_i||/||b||\ttime(sec)\n"); for (int32_t i = 1; i <= numiter/oh->verbose; i++) printf ("%d\t%1.3e\t%1.3e\n", i*oh->verbose, dat1[i], dat2[i]); printf ("## total time = %1.3e\n", t3-t1); if (oh->trueres) printf ("## -> including true residual computation\n"); printf ("## iter-part time = %1.3e\n", t3-t2); if (oh->trueres) printf ("## -> including true residual computation\n"); printf ("## total iter = %d\n", numiter); printf ("## time per iter = %1.3e\n", (t3-t2)/numiter); printf ("## ||r_i||/||b|| = %1.3e\n", resi/nrmb); printf ("## ||b-Ax||/||b|| = %1.3e\n", res0/nrmb); // ================================= printf ("%1.3e\t", tloc_SpMV_SpMM_total); printf ("%1.3e\t", tloc_DOT_NRM2_total); printf ("%1.3e\t", tloc_AXPY_SCAL_total); printf ("%1.3e\t", tloc_Sum_total); printf ("%1.3e\t", tloc_SplitVec_total); printf ("%1.3e\t", tloc_SplitMat_total); printf ("%1.3e\t#(SpMV/MM, DOT/NRM, AXPY/SCAL, Sum, SpltVec, SpltMat, Other)\n", tloc_Other_total); // ================================= printf ("%d\t%1.3e\t(%a)\t%1.3e\t(%a)\t%1.3e\t%1.3e\t#(iter, res, trueres, ttl-time, itr-prt-time)\n", numiter, resi/nrmb, resi/nrmb, res0/nrmb, res0/nrmb, t3-t1, t3-t2); free (dat1); free (dat2); } free (vecT); free (vecR); free (vecP); free (vecQ); // ================================= oh->t_SplitMat_total = 0.; oh->t_SplitVec_total = 0.; oh->t_Sum_total = 0.; oh->t_AXPY_SCAL_total = 0.; oh->t_DOT_NRM2_total = 0.; oh->t_SpMV_SpMM_total = 0.; // reset parameters oh->memMaskSplitA = 0; // disable pre-split of matA oh->splitShift = 1; // default (no-Splitshift) // ================================= return 0; } ozblas-1.2a17/src/cpu/ozblas_common.h0000644000175200017520000001271013736547514020062 0ustar mukunokimukunoki#define MIN(a, b) ((a) < (b) ? (a) : (b)) #define MAX(a, b) ((a) > (b) ? (a) : (b)) // setting #define DefaultWorkSize 1e9 // 1GB #define NumSplitDefaultMax 20 #include #include #define __STDC_FORMAT_MACROS #include #include #include #ifdef NVBLAS #include #endif #ifdef MKL #include #else #include #endif #include #include #include "../../include/ozblas.h" #include "ozblas_internal.h" #if defined (PREC_S) #define FP_DTYPE float #define FP_CTYPE float #define UNIT 24 #define OzCG ozblasScg #define OzDOT ozblasSdot #define OzAXPY ozblasSaxpy #define OzGEMV ozblasSgemv #define OzGEMM ozblasSgemm #define OzNRM2 ozblasSnrm2 #define OzCSRMV ozblasScsrmv #define OzCSRMVSplitA ozblasScsrmvSplitA #define RCBLAS_AXPY cblas_saxpy #define RCBLAS_DOT cblas_sdot #define RCBLAS_GEMV cblas_sgemv #define RCBLAS_GEMM cblas_sgemm #define RNVBLAS_GEMM sgemm #define RCBLAS_NRM2 cblas_snrm2 #define RMKL_CSRMV mkl_scsrmv #define NVBLAS_GEMM sgemm #define NVBLAS_GEMMBATCH sgemm_batch #define MKL_OMATCOPY mkl_somatcopy #define MKL_CSRMV mkl_scsrmv #define MKL_CSRMM mkl_scsrmm #define CBLAS_OMATCOPY cblas_somatcopy #define CBLAS_AXPY cblas_saxpy #define CBLAS_DOT cblas_sdot #define CBLAS_GEMV cblas_sgemv #define CBLAS_GEMM cblas_sgemm #define CBLAS_GEMMBATCH cblas_sgemm_batch #define CBLAS_NRM2 cblas_snrm2 #define CBLAS_SCAL cblas_sscal #define CBLAS_ASUM cblas_sasum #define CBLAS_IAMAX cblas_isamax #define ozblasMallocSplitMat ozblasSMallocSplitMat #define ozblasMallocSplitMatSparse ozblasSMallocSplitMatSparse #define ozblasMallocResultMat ozblasSMallocResultMat #define ozblasMallocSplitVec ozblasSMallocSplitVec #define ozblasMallocResultVec ozblasSMallocResultVec #define ozblasSplitMat ozblasSSplitMat #define ozblasSplitMatSparse ozblasSSplitMatSparse #define ozblasSplitShiftMat ozblasSSplitShiftMat #define ozblasSplitShiftMatSparse ozblasSSplitShiftMatSparse #define ozblasSumMat ozblasSSumMat #define ozblasSumMatNpara ozblasSSumMatNpara #define ozblasPrintVec ozblasSPrintVec #define ozblasCopyVec ozblasSCopyVec #elif defined (PREC_D) #define FP_DTYPE double #define FP_CTYPE double #define UNIT 53 #define OzCG ozblasDcg #define OzDOT ozblasDdot #define OzAXPY ozblasDaxpy #define OzGEMV ozblasDgemv #define OzGEMM ozblasDgemm #define OzNRM2 ozblasDnrm2 #define OzCSRMV ozblasDcsrmv #define OzCSRMVSplitA ozblasDcsrmvSplitA #define RCBLAS_AXPY cblas_daxpy #define RCBLAS_DOT cblas_ddot #define RCBLAS_GEMV cblas_dgemv #define RCBLAS_GEMM cblas_dgemm #define RNVBLAS_GEMM dgemm #define RCBLAS_NRM2 cblas_dnrm2 #define RMKL_CSRMV mkl_dcsrmv #define NVBLAS_GEMM dgemm #define NVBLAS_GEMMBATCH dgemm_batch #define MKL_OMATCOPY mkl_domatcopy #define MKL_CSRMV mkl_dcsrmv #define MKL_CSRMM mkl_dcsrmm #define CBLAS_OMATCOPY cblas_domatcopy #define CBLAS_AXPY cblas_daxpy #define CBLAS_DOT cblas_ddot #define CBLAS_GEMV cblas_dgemv #define CBLAS_GEMM cblas_dgemm #define CBLAS_GEMMBATCH cblas_dgemm_batch #define CBLAS_NRM2 cblas_dnrm2 #define CBLAS_SCAL cblas_dscal #define CBLAS_ASUM cblas_dasum #define CBLAS_IAMAX cblas_idamax #define ozblasMallocSplitMat ozblasDMallocSplitMat #define ozblasMallocSplitMatSparse ozblasDMallocSplitMatSparse #define ozblasMallocResultMat ozblasDMallocResultMat #define ozblasMallocSplitVec ozblasDMallocSplitVec #define ozblasMallocResultVec ozblasDMallocResultVec #define ozblasSplitMat ozblasDSplitMat #define ozblasSplitMatSparse ozblasDSplitMatSparse #define ozblasSplitShiftMat ozblasDSplitShiftMat #define ozblasSplitShiftMatSparse ozblasDSplitShiftMatSparse #define ozblasSumMat ozblasDSumMat #define ozblasSumMatNpara ozblasDSumMatNpara #define ozblasPrintVec ozblasDPrintVec #define ozblasCopyVec ozblasDCopyVec #elif defined (PREC_SD) #define FP_DTYPE float #define FP_CTYPE double #define UNIT 53 #define OzCG ozblasSDcg #define OzDOT ozblasSDdot #define OzAXPY ozblasSDaxpy #define OzGEMV ozblasSDgemv #define OzGEMM ozblasSDgemm #define OzNRM2 ozblasSDnrm2 #define OzCSRMV ozblasSDcsrmv #define OzCSRMVSplitA ozblasSDcsrmvSplitA #define RCBLAS_AXPY cblas_saxpy #define RCBLAS_DOT cblas_sdot #define RCBLAS_GEMV cblas_sgemv #define RCBLAS_GEMM cblas_sgemm #define RNVBLAS_GEMM sgemm #define RCBLAS_NRM2 cblas_snrm2 #define RMKL_CSRMV mkl_scsrmv #define NVBLAS_GEMM dgemm #define NVBLAS_GEMMBATCH dgemm_batch #define MKL_OMATCOPY mkl_domatcopy #define MKL_CSRMV mkl_dcsrmv #define MKL_CSRMM mkl_dcsrmm #define CBLAS_OMATCOPY cblas_domatcopy #define CBLAS_AXPY cblas_daxpy #define CBLAS_DOT cblas_ddot #define CBLAS_GEMV cblas_dgemv #define CBLAS_GEMM cblas_dgemm #define CBLAS_GEMMBATCH cblas_dgemm_batch #define CBLAS_NRM2 cblas_dnrm2 #define CBLAS_SCAL cblas_sscal // <- 's' #define CBLAS_ASUM cblas_dasum #define CBLAS_IAMAX cblas_idamax #define ozblasMallocSplitMat ozblasSDMallocSplitMat #define ozblasMallocSplitMatSparse ozblasSDMallocSplitMatSparse #define ozblasMallocResultMat ozblasSDMallocResultMat #define ozblasMallocSplitVec ozblasSDMallocSplitVec #define ozblasMallocResultVec ozblasSDMallocResultVec #define ozblasSplitMat ozblasSDSplitMat #define ozblasSplitMatSparse ozblasSDSplitMatSparse #define ozblasSplitShiftMat ozblasSDSplitShiftMat #define ozblasSplitShiftMatSparse ozblasSDSplitShiftMatSparse #define ozblasSumMat ozblasSDSumMat #define ozblasSumMatNpara ozblasSDSumMatNpara #define ozblasPrintVec ozblasSDPrintVec #define ozblasCopyVec ozblasSDCopyVec #endif ozblas-1.2a17/src/cpu/ozblas_internal.h0000644000175200017520000001771713736550764020423 0ustar mukunokimukunoki#ifndef OZBLAS_INTERNAL_H #define OZBLAS_INTERNAL_H double timer (); int getPitchSize (int n); void counterInit (ozblasHandle_t *oh); uint32_t memCheck (ozblasHandle_t *oh); #if defined (PREC_D) void ozblasDMallocSplitMat ( ozblasHandle_t *oh, const int32_t m, const int32_t n, const uint32_t numSplitArraysMax, double **hstSplit, double **hstMax, int32_t &lds); void ozblasDMallocSplitMatSparse ( ozblasHandle_t *oh, const int32_t m, const int32_t nnz, const uint32_t numSplitArraysMax, double **devSplit, double **devMax, int32_t &lds); void ozblasDMallocResultMat ( ozblasHandle_t *oh, const int32_t m, const int32_t n, const uint32_t numSplitArraysMax, double **hstSplit, int32_t &lds); void ozblasDMallocSplitVec ( ozblasHandle_t *oh, const int32_t n, const uint32_t numSplitArraysMax, double **hstSplit, double **hstMax, int32_t &lds); void ozblasDMallocResultVec ( ozblasHandle_t *oh, const int32_t n, const uint32_t numSplitArraysMax, double **hstSplit, int32_t &lds); void ozblasDSplitMat ( ozblasHandle_t *oh, const char trans, const int32_t m, const int32_t n, const double *hstInput, const int32_t ld, double *hstSplit, double *hstMax, const int32_t lds, uint32_t &numSplitArrays, uint32_t &lastFlag); void ozblasDSplitMatSparse ( ozblasHandle_t *oh, const char trans, const int32_t m, const double *hstInput, const int32_t *hstRowptr, double *hstSplit, double *hstMax, const int32_t lds, uint32_t &numSplitArrays, uint32_t &lastFlag); void ozblasDSplitShiftMat ( ozblasHandle_t *oh, const char trans, const int32_t m, const int32_t n, const double *hstInput, const int32_t ld, double *hstSplit, double *hstMax, const int32_t lds, uint32_t &numSplitArrays, uint32_t &lastFlag); void ozblasDSplitShiftMatSparse ( ozblasHandle_t *oh, const char trans, const int32_t m, const double *hstInput, const int32_t *hstRowptr, double *hstSplit, double *hstMax, const int32_t lds, uint32_t &numSplitArrays, uint32_t &lastFlag); void ozblasDSumMat ( const int32_t m, const int32_t n, double *hstCsplit, const int32_t llsc, const int32_t ldsc, const int32_t numsplit, double *hstC, const int32_t ldc, const double alpha, const double beta, const uint32_t summode); void ozblasDSumMatNpara ( const int32_t m, const int32_t n, double *hstCsplit, const int32_t llsc, const int32_t ldsc, const int32_t numsplit, double *hstC, const int32_t ldc, const double alpha, const double beta, const uint32_t summode); void ozblasDPrintVec ( const int32_t n, const double *devC); void ozblasDCopyVec ( const size_t n, const double *hstA, double *hstB); #elif defined (PREC_S) void ozblasSMallocSplitMat ( ozblasHandle_t *oh, const int32_t m, const int32_t n, const uint32_t numSplitArraysMax, float **hstSplit, float **hstMax, int32_t &lds); void ozblasSMallocSplitMatSparse ( ozblasHandle_t *oh, const int32_t m, const int32_t nnz, const uint32_t numSplitArraysMax, float **devSplit, float **devMax, int32_t &lds); void ozblasSMallocResultMat ( ozblasHandle_t *oh, const int32_t m, const int32_t n, const uint32_t numSplitArraysMax, float **hstSplit, int32_t &lds); void ozblasSMallocSplitVec ( ozblasHandle_t *oh, const int32_t n, const uint32_t numSplitArraysMax, float **hstSplit, float **hstMax, int32_t &lds); void ozblasSMallocResultVec ( ozblasHandle_t *oh, const int32_t n, const uint32_t numSplitArraysMax, float **hstSplit, int32_t &lds); void ozblasSSplitMat ( ozblasHandle_t *oh, const char trans, const int32_t m, const int32_t n, const float *hstInput, const int32_t ld, float *hstSplit, float *hstMax, const int32_t lds, uint32_t &numSplitArrays, uint32_t &lastFlag); void ozblasSSplitMatSparse ( ozblasHandle_t *oh, const char trans, const int32_t m, const float *hstInput, const int32_t *hstRowptr, float *hstSplit, float *hstMax, const int32_t lds, uint32_t &numSplitArrays, uint32_t &lastFlag); void ozblasSSplitShiftMat ( ozblasHandle_t *oh, const char trans, const int32_t m, const int32_t n, const float *hstInput, const int32_t ld, float *hstSplit, float *hstMax, const int32_t lds, uint32_t &numSplitArrays, uint32_t &lastFlag); void ozblasSSplitShiftMatSparse ( ozblasHandle_t *oh, const char trans, const int32_t m, const float *hstInput, const int32_t *hstRowptr, float *hstSplit, float *hstMax, const int32_t lds, uint32_t &numSplitArrays, uint32_t &lastFlag); void ozblasSSumMat ( const int32_t m, const int32_t n, float *hstCsplit, const int32_t llsc, const int32_t ldsc, const int32_t numsplit, float *hstC, const int32_t ldc, const float alpha, const float beta, const uint32_t summode); void ozblasSSumMatNpara ( const int32_t m, const int32_t n, float *hstCsplit, const int32_t llsc, const int32_t ldsc, const int32_t numsplit, float *hstC, const int32_t ldc, const float alpha, const float beta, const uint32_t summode); void ozblasSPrintVec ( const int32_t n, const float *devC); void ozblasSCopyVec ( const size_t n, const float *hstA, float *hstB); #elif defined (PREC_SD) void ozblasSDMallocSplitMat ( ozblasHandle_t *oh, const int32_t m, const int32_t n, const uint32_t numSplitArraysMax, double **hstSplit, double **hstMax, int32_t &lds); void ozblasSDMallocSplitMatSparse ( ozblasHandle_t *oh, const int32_t m, const int32_t nnz, const uint32_t numSplitArraysMax, double **devSplit, double **devMax, int32_t &lds); void ozblasSDMallocResultMat ( ozblasHandle_t *oh, const int32_t m, const int32_t n, const uint32_t numSplitArraysMax, double **hstSplit, int32_t &lds); void ozblasSDMallocSplitVec ( ozblasHandle_t *oh, const int32_t n, const uint32_t numSplitArraysMax, double **hstSplit, double **hstMax, int32_t &lds); void ozblasSDMallocTmpMat ( ozblasHandle_t *oh, const int32_t m, const int32_t n, double **hstSplit, int32_t &lds); void ozblasSDMallocTmpVec ( ozblasHandle_t *oh, const int32_t n, double **hstSplit, int32_t &lds); void ozblasSDCopyMat ( const int32_t m, const int32_t n, const float *hstA, const int32_t lda, double *hstB, const int32_t ldb); void ozblasDCopyMat ( const int32_t m, const int32_t n, const double *hstA, const int32_t lda, double *hstB, const int32_t ldb); void ozblasSDMallocResultVec ( ozblasHandle_t *oh, const int32_t n, const uint32_t numSplitArraysMax, double **hstSplit, int32_t &lds); void ozblasSDSplitMat ( ozblasHandle_t *oh, const char trans, const int32_t m, const int32_t n, const double *hstInput, const int32_t ld, double *hstSplit, double *hstMax, const int32_t lds, uint32_t &numSplitArrays, uint32_t &lastFlag); void ozblasSDSplitMatSparse ( ozblasHandle_t *oh, const char trans, const int32_t m, const double *hstInput, const int32_t *hstRowptr, double *hstSplit, double *hstMax, const int32_t lds, uint32_t &numSplitArrays, uint32_t &lastFlag); void ozblasSDSplitShiftMat ( ozblasHandle_t *oh, const char trans, const int32_t m, const int32_t n, const double *hstInput, const int32_t ld, double *hstSplit, double *hstMax, const int32_t lds, uint32_t &numSplitArrays, uint32_t &lastFlag); void ozblasSDSplitShiftMatSparse ( ozblasHandle_t *oh, const char trans, const int32_t m, const double *hstInput, const int32_t *hstRowptr, double *hstSplit, double *hstMax, const int32_t lds, uint32_t &numSplitArrays, uint32_t &lastFlag); void ozblasSDSumMat ( const int32_t m, const int32_t n, double *hstCsplit, const int32_t llsc, const int32_t ldsc, const int32_t numsplit, float *hstC, const int32_t ldc, const float alpha, const float beta, const uint32_t summode); void ozblasSDSumMatNpara ( const int32_t m, const int32_t n, double *hstCsplit, const int32_t llsc, const int32_t ldsc, const int32_t numsplit, float *hstC, const int32_t ldc, const float alpha, const float beta, const uint32_t summode); void ozblasSDPrintVec ( const int32_t n, const float *devC); void ozblasSDCopyVec ( const size_t n, const float *hstA, float *hstB); void ozblasDCopyVec ( const size_t n, const double *hstA, double *hstB); #endif //void ozblasInitVecDevice ( const int32_t n, int64_t *hstA); CBLAS_TRANSPOSE ToCblasOp (char tran); char FromCublasOp (CBLAS_TRANSPOSE tran); void PrintMat ( const int32_t m, const int32_t n, const double *devC, const int32_t ldd); void PrintMatInt ( const int32_t m, const int32_t n, const int32_t *devC, const int32_t ldd); #endif ozblas-1.2a17/src/cpu/ozblas_auxiliary.cpp0000644000175200017520000001126713736550206021132 0ustar mukunokimukunoki#include "ozblas_common.h" // ========================================= // Print floating-point value with bit representation // ========================================= typedef union{ FP_CTYPE d; uint64_t i; } d_and_i; void printBits (FP_CTYPE val) { d_and_i di; di.d = val; // sign printf ("%zu", (di.i >> 63) & 1); printf ("|"); // exponent for (int i = 62; i >= 62-10; i--) printf ("%zu", (di.i >> i) & 1); printf ("|"); // fraction for (int i = 62-11; i >= 0; i--) printf ("%zu", (di.i >> i) & 1); printf (" : "); printf ("%+1.18e", val); } // ========================================= // PrintMat (for debugging) // ========================================= void PrintMat ( const int32_t m, const int32_t n, const FP_CTYPE *devC, const int32_t ldd ) { FP_CTYPE tmp; int32_t i, j, ldh; ldh = m; printf ("\n"); for (j = 0; j < n; j++) { for (i = 0; i < m; i++) { tmp = devC[j*ldh+i]; if (tmp != 0) { printf ("[%2d,%2d] ", j, i); printBits (tmp); printf ("\n"); } } } printf ("\n"); } void PrintMatInt ( const int32_t m, const int32_t n, const int32_t *devC, const int32_t ldd ) { int32_t tmp; int32_t i, j, ldh; ldh = m; printf ("\n"); for (j = 0; j < n; j++) { for (i = 0; i < m; i++) { tmp = devC[j*ldh+i]; // if (tmp != 0) { printf ("[%2d,%2d] %d", j, i, tmp); printf ("\n"); // } } } printf ("\n"); } void ozblasPrintVec ( const int32_t n, const FP_DTYPE *devC ) { for (int32_t i = 0; i < n; i++) printf ("%1.18e\n", devC[i]); } // ========================================= // Matrix Allocation // ========================================= void ozblasMallocSplitVec ( ozblasHandle_t *oh, const int n, const uint32_t numSplitArraysMax, FP_CTYPE **devSplit, FP_CTYPE **devMax, int &lds ) { // oh->workSizeBytes; // size of work memory [Bytes] // oh->devWork; // the start point of the work memory // devXsplit starts from the end of devXsplit0 lds = getPitchSize (n); devSplit[0] = (FP_CTYPE*)(oh->devWork + oh->memAddr); oh->memAddr += sizeof(FP_CTYPE) * lds * numSplitArraysMax; int ldv = 1; // '+1': devMax needs work areas staring from [getPitchSize(1)] devMax[0] = (FP_CTYPE*)(oh->devWork + oh->memAddr); oh->memAddr += sizeof(FP_CTYPE) * ldv; memCheck (oh); } void ozblasMallocResultVec ( ozblasHandle_t *oh, const int n, const uint32_t numSplitArraysMax, FP_CTYPE **devSplit, int &lds ) { lds = getPitchSize (n); devSplit[0] = (FP_CTYPE*)(oh->devWork + oh->memAddr); oh->memAddr += sizeof(FP_CTYPE) * lds * numSplitArraysMax * numSplitArraysMax; memCheck (oh); } void ozblasMallocSplitMat ( ozblasHandle_t *oh, const int m, const int n, const uint32_t numSplitArraysMax, FP_CTYPE **devSplit, FP_CTYPE **devMax, int &lds ) { lds = getPitchSize (m); devSplit[0] = (FP_CTYPE*)(oh->devWork + oh->memAddr); oh->memAddr += sizeof(FP_CTYPE) * lds * n * (numSplitArraysMax + 1); // +1 is for output (work) devMax[0] = (FP_CTYPE*)(oh->devWork + oh->memAddr); oh->memAddr += sizeof(FP_CTYPE) * lds; memCheck (oh); } #if defined (PREC_SD) void ozblasSDMallocTmpVec ( ozblasHandle_t *oh, const int n, FP_CTYPE **devTmp, int &lds ) { lds = getPitchSize (n); devTmp[0] = (FP_CTYPE*)(oh->devWork + oh->memAddr); oh->memAddr += sizeof(FP_CTYPE) * lds; } void ozblasSDMallocTmpMat ( ozblasHandle_t *oh, const int m, const int n, FP_CTYPE **devTmp, int &lds ) { lds = getPitchSize (m); devTmp[0] = (FP_CTYPE*)(oh->devWork + oh->memAddr); oh->memAddr += sizeof(FP_CTYPE) * lds * n; // +1 is for output (work) } void ozblasSDCopyMat ( const int m, const int n, const FP_DTYPE *devA, const int lda, FP_CTYPE *devB, const int ldb ) { int i, j; #pragma omp parallel for private (i) for (j = 0; j < n; j++){ for (i = 0; i < m; i++) devB[j*ldb+i] = devA[j*lda+i]; } } void ozblasCopyVec ( const int n, const FP_CTYPE *devA, FP_CTYPE *devB ) { int i; #pragma omp parallel for for (i = 0; i < n; i++) devB[i] = devA[i]; } #endif void ozblasMallocResultMat ( ozblasHandle_t *oh, const int m, const int n, const uint32_t numSplitArraysMax, FP_CTYPE **devSplit, int &lds ) { lds = getPitchSize (m); devSplit[0] = (FP_CTYPE*)(oh->devWork + oh->memAddr); oh->memAddr += sizeof(FP_CTYPE) * lds * n * numSplitArraysMax * numSplitArraysMax; memCheck (oh); } void ozblasMallocSplitMatSparse ( ozblasHandle_t *oh, const int m, const int nnz, const uint32_t numSplitArraysMax, FP_CTYPE **devSplit, FP_CTYPE **devMax, int &lds ) { lds = getPitchSize (nnz); devSplit[0] = (FP_CTYPE*)(oh->devWork + oh->memAddr); oh->memAddr += sizeof(FP_CTYPE) * lds * (numSplitArraysMax + 1); // +1 is for output (work) devMax[0] = (FP_CTYPE*)(oh->devWork + oh->memAddr); oh->memAddr += sizeof(FP_CTYPE) * getPitchSize(m); } ozblas-1.2a17/src/cpu/ozblas_csrmv.cpp0000644000175200017520000001766013736551053020261 0ustar mukunokimukunoki#include "ozblas_common.h" int32_t OzCSRMV ( ozblasHandle_t *oh, const char tranA, const int32_t m, const int32_t n, const int32_t nnz, const FP_DTYPE alpha, const char *descrA, const FP_DTYPE *devA, const int32_t *devArowptr, const int32_t *devAcolind, const FP_DTYPE *devB, const FP_DTYPE beta, FP_DTYPE *devC ) { counterInit (oh); if (oh->reproModeFlag == 0 && oh->numSplitArraysMax == 1) { double t0 = timer(); #if defined (PREC_SD) printf ("not supported\n"); exit(1); #else RMKL_CSRMV (&tranA, &m, &n, &alpha, descrA, devA, devAcolind, devArowptr, devArowptr+1, devB, &beta, devC); #endif oh->t_SpMV_SpMM_total += timer() - t0; return 0; } double t1, t0 = timer(); uint32_t numSplitArraysMax = (oh->numSplitArraysMax > 0) ? oh->numSplitArraysMax : NumSplitDefaultMax; int32_t ldas, ldbs, ldcs; FP_CTYPE fone = 1., fzero = 0.; FP_CTYPE done = 1; FP_CTYPE *devASplit; FP_CTYPE *devAmax, *devBmax; FP_CTYPE *devBSplit, *devCSplit; int32_t mbk = m; // currently blocking is not supported if (mbk != m && oh->memMaskSplitA) { // when blocking is applied, memMaskSplitA cannot be used printf ("Blocking is not supported with WithOutSplitA mode\n"); exit(1); } if (oh->memMaskSplitA != 0) oh->memAddr = 0; ozblasMallocSplitVec (oh, nnz, numSplitArraysMax, &devASplit, &devAmax, ldas); // this is needed even when memMaskSplitA if (oh->memMaskSplitA != 0) oh->memMaskSplitA = oh->memAddr; ozblasMallocSplitVec (oh, n, numSplitArraysMax, &devBSplit, &devBmax, ldbs); ldcs = getPitchSize (mbk); devCSplit = (FP_CTYPE*)(oh->devWork + oh->memAddr); oh->memAddr += sizeof(FP_CTYPE) * ldcs * numSplitArraysMax * numSplitArraysMax; #if defined (PREC_SD) int32_t ldat, ldbt; FP_CTYPE *devATmp, *devBTmp; if (oh->memMaskSplitA == 0) { ozblasSDMallocTmpVec (oh, nnz, &devATmp, ldat); ozblasSDCopyMat (nnz, 1, devA, nnz, devATmp, ldat); } ozblasSDMallocTmpVec (oh, n, &devBTmp, ldbt); ozblasSDCopyMat (n, 1, devB, n, devBTmp, ldbt); #endif memCheck (oh); for (int32_t im = 0; im < ceil((float)m/mbk); im++) { int32_t mbk_ = (m-mbk*im >= mbk) ? mbk : m-mbk*im; // Split of A ----------------------------------- t1 = timer(); if (oh->memMaskSplitA == 0) { oh->numSplitArraysA = numSplitArraysMax; #if defined (PREC_SD) ozblasSplitShiftMatSparse (oh, 'n', m, devATmp, devArowptr, devASplit, devAmax, ldas, oh->numSplitArraysA, oh->lastFlagA); #else ozblasSplitShiftMatSparse (oh, 'n', m, devA, devArowptr, devASplit, devAmax, ldas, oh->numSplitArraysA, oh->lastFlagA); #endif } else { devASplit = (FP_CTYPE*)devA; oh->numSplitArraysA = oh->numSplitArraysA_; } oh->t_SplitA += timer() - t1; // Split of B ----------------------------------- t1 = timer(); oh->numSplitArraysB = numSplitArraysMax; #if defined (PREC_SD) ozblasSplitShiftMat (oh, 'r', n, 1, devBTmp, n, devBSplit, devBmax, ldbs, oh->numSplitArraysB, oh->lastFlagB); #else ozblasSplitShiftMat (oh, 'r', n, 1, devB, n, devBSplit, devBmax, ldbs, oh->numSplitArraysB, oh->lastFlagB); #endif oh->t_SplitB += timer() - t1; // Compute -------------------------------------- t1 = timer(); int32_t numB, ia, ic = 0; uint32_t maxlevel = (oh->numSplitArraysA-1) + (oh->numSplitArraysB-1); if (oh->fastModeFlag) { if (oh->numSplitArraysMax == 0) maxlevel = MIN (oh->numSplitArraysA-1, oh->numSplitArraysB-1); else maxlevel = oh->numSplitArraysMax-1; } FP_CTYPE *ptrA, *ptrB, *ptrC; ptrC = devCSplit; for (ia = 0; ia < MIN (maxlevel+1, oh->numSplitArraysA); ia++) { ptrA = devASplit+ldas*ia; if (oh->useGemmFlag) { // Note: mkl_dcsrmm seems to use Row-major storage for dense matrices. numB = MIN (oh->numSplitArraysB, maxlevel+1 - ia); ptrB = devBSplit; MKL_CSRMM (&tranA, &mbk_, &numB, &n, &done, descrA, ptrA, devAcolind, devArowptr, devArowptr+1, ptrB, &ldbs, &fzero, ptrC, &ldcs); ptrC += ldcs*numB; ic += numB; } else { for (uint32_t ib = 0; ib < oh->numSplitArraysB; ib++) { if (ia + ib <= maxlevel) { ptrB = devBSplit+ldbs*ib; ptrC = devCSplit+ldcs*ic; MKL_CSRMV (&tranA, &mbk_, &n, &fone, descrA, ptrA, devAcolind, devArowptr, devArowptr+1, ptrB, &fzero, ptrC); ic++; } } } } oh->numSplitArraysC = ic; oh->t_comp += timer() - t1; // Sum ----------------------------------------- t1 = timer(); ozblasSumMat (mbk_, 1, devCSplit, ldcs, ldcs, oh->numSplitArraysC, &devC[im*mbk], ldcs, alpha, beta, oh->sumModeFlag); oh->t_sum += timer() - t1; } oh->t_total = timer() - t0; // for CG, time // ================================= oh->t_SplitMat_total += oh->t_SplitA; oh->t_SplitVec_total += oh->t_SplitB; oh->t_Sum_total += oh->t_sum; oh->t_AXPY_SCAL_total += 0.; oh->t_DOT_NRM2_total += 0.; oh->t_SpMV_SpMM_total += oh->t_comp; // ================================= return 0; } int32_t OzCSRMVSplitA ( ozblasHandle_t *oh, const char tranA, const int32_t m, const int32_t n, const int32_t nnz, const char *descrA, const FP_CTYPE *devA, const int32_t *devArowptr, const int32_t *devAcolind, FP_CTYPE **devASplit ) { counterInit (oh); uint32_t numSplitArraysMax = (oh->numSplitArraysMax > 0) ? oh->numSplitArraysMax : NumSplitDefaultMax; uint32_t numSplitArraysMaxOrig = oh->numSplitArraysMax; int32_t ldas; FP_CTYPE *devAmax; #if defined (PREC_SD) int32_t ldat; FP_CTYPE *devATmp; ozblasSDMallocTmpVec (oh, nnz, &devATmp, ldat); ozblasDCopyMat (nnz, 1, devA, nnz, devATmp, ldat); #endif ozblasMallocSplitMatSparse (oh, m, nnz, numSplitArraysMax, &devASplit[0], &devAmax, ldas); memCheck (oh); oh->memMaskSplitA = oh->memAddr; double t1 = timer(); if (oh->reproModeFlag == 0 && oh->numSplitArraysMax == 1) { #if defined (PREC_SD) ozblasDCopyVec (nnz, devA, devASplit[0]); #else ozblasCopyVec (nnz, devA, devASplit[0]); #endif } else { // Split of A ----------------------------------- oh->numSplitArraysA = numSplitArraysMax; #if defined (PREC_SD) ozblasSplitMatSparse (oh, 'n', m, devATmp, devArowptr, devASplit[0], devAmax, ldas, oh->numSplitArraysA, oh->lastFlagA); #else ozblasSplitMatSparse (oh, 'n', m, devA, devArowptr, devASplit[0], devAmax, ldas, oh->numSplitArraysA, oh->lastFlagA); #endif // shiftSize-tuning // ------------------------------------------------------------------------ if (numSplitArraysMaxOrig == 0) { // tuning is possible only when full-splitting (d=0) printf ("## CSRMV: << shift-size tuning >> num.split = %d -> ", oh->numSplitArraysA); oh->splitShift = 1; int32_t numSplitArraysAOld = oh->numSplitArraysA; do { numSplitArraysAOld = oh->numSplitArraysA; oh->splitShift *= 2; // try with new shift-size oh->numSplitArraysA = numSplitArraysMax; #if defined (PREC_SD) ozblasSplitShiftMatSparse (oh, 'n', m, devATmp, devArowptr, devASplit[0], devAmax, ldas, oh->numSplitArraysA, oh->lastFlagA); #else ozblasSplitShiftMatSparse (oh, 'n', m, devA, devArowptr, devASplit[0], devAmax, ldas, oh->numSplitArraysA, oh->lastFlagA); #endif // if numSplit increased, stop } while (numSplitArraysAOld == oh->numSplitArraysA); // do again with the optimal shift-size oh->splitShift /= 2; oh->numSplitArraysA = numSplitArraysMax; #if defined (PREC_SD) ozblasSplitShiftMatSparse (oh, 'n', m, devATmp, devArowptr, devASplit[0], devAmax, ldas, oh->numSplitArraysA, oh->lastFlagA); #else ozblasSplitShiftMatSparse (oh, 'n', m, devA, devArowptr, devASplit[0], devAmax, ldas, oh->numSplitArraysA, oh->lastFlagA); #endif printf ("%d (with splitShift = %d)\n", oh->numSplitArraysA, oh->splitShift); } // ------------------------------------------------------------------------ oh->numSplitArraysA_ = oh->numSplitArraysA; } oh->t_SplitA += timer() - t1; // for CG, time // ================================= oh->t_SplitMat_total += oh->t_SplitA; oh->t_SplitVec_total += 0.; oh->t_Sum_total += 0.; oh->t_AXPY_SCAL_total += 0.; oh->t_DOT_NRM2_total += 0.; oh->t_SpMV_SpMM_total += 0.; // ================================= return 0; } ozblas-1.2a17/testing/0000755000175200017520000000000013736552053015137 5ustar mukunokimukunokiozblas-1.2a17/testing/testing_level2.cpp0000644000175200017520000001415613725177602020601 0ustar mukunokimukunoki#define ROUTINE_FLOPS(m,n,k) (2*m*n+3*m) #define ROUTINE_BYTES(m,n,k,s) ((2*m+n+m*n)*s) #include "testing_common.h" #include "testing_common.cpp" int32_t main (int32_t argc, char **argv) { int32_t i; int32_t lda_hst; int32_t cda_dev, cda_hst; int32_t rda_dev, rda_hst; int32_t vlx_dev, vlx_hst; int32_t vly_dev, vly_hst; double t0, t1, sec; mpfr_set_default_prec(MPFR_PREC); mpreal::set_default_prec(MPFR_PREC); // testing setup ------------------------------ testingHandle_t th; testingCreate (argc, argv, &th); print_info1 (&th); // -------------------------------------------- // library setup ------------------------------ #if defined (CUOZBLAS) cuozblasHandle_t ha; cuozblasCreate (&ha, WORK_MEM_SIZE, th.numSplitArraysMax, th.fastmode, th.repromode, th.summode); cuozblasProfilingSetting (&ha, th.useGemmFlag, th.useMyGemmFlag, th.useBatchedGemmFlag); #else ozblasHandle_t ha; ozblasCreate (&ha, WORK_MEM_SIZE, th.numSplitArraysMax, th.fastmode, th.repromode, th.summode); ozblasProfilingSetting (&ha, th.useGemmFlag, th.useMyGemmFlag, th.useBatchedGemmFlag); #endif // -------------------------------------------- // memory setup ------------------------------- dim_hst_setup (&th); rda_hst = th.dim_stop; cda_hst = th.dim_stop; vlx_hst = th.dim_stop; vly_hst = th.dim_stop; if (th.tranA == 'N' || th.tranA == 'n') { if (th.dim_n_const > 0) vlx_hst = th.dim_n_const; if (th.dim_m_const > 0) vly_hst = th.dim_m_const; } else { if (th.dim_m_const > 0) vlx_hst = th.dim_m_const; if (th.dim_n_const > 0) vly_hst = th.dim_n_const; } if (th.dim_m_const > 0) rda_hst = th.dim_m_const; if (th.dim_n_const > 0) cda_hst = th.dim_n_const; lda_hst = rda_hst; // malloc host memory FP_TYPE alpha, beta; FP_TYPE *hst_A = new FP_TYPE[lda_hst * cda_hst]; FP_TYPE *hst_X = new FP_TYPE[vlx_hst * th.incx]; FP_TYPE *hst_Y = new FP_TYPE[vly_hst * th.incy]; FP_TYPE *hst_Y_t = new FP_TYPE[vly_hst * th.incy]; // initialize (1:val, 2:phi, 3:erange) mublasInitMat (1, 1, 1, &alpha, 1., 1); mublasInitMat (1, 1, 1, &beta, 0., 1); mublasInitMat (lda_hst, cda_hst, lda_hst, hst_A, th.phi, 3); mublasInitMat (vlx_hst * th.incx, 1, 0, hst_X, th.phi, 3); mublasInitMat (vly_hst * th.incy, 1, 0, hst_Y, th.phi, 3); // -------------------------------------------- print_info2 (&th); // evaluation --------------------------------- dim_dev_setup (&th); while (1) { if ((th.dim_m_const == 0 && th.dim_m_dev > th.dim_stop) || (th.dim_n_const == 0 && th.dim_n_dev > th.dim_stop)) break; // dim setup if (th.tranA == 'N' || th.tranA == 'n') { vlx_dev = th.dim_n_dev; vly_dev = th.dim_m_dev; } else { vlx_dev = th.dim_m_dev; vly_dev = th.dim_n_dev; } rda_dev = th.dim_m_dev; cda_dev = th.dim_n_dev; #if defined (CUOZBLAS) // malloc device memory FP_TYPE *dev_A, *dev_X, *dev_Y; size_t pitch; cudaMallocPitch ((void **) &dev_A, &pitch, sizeof(FP_TYPE) * rda_dev, cda_dev); int32_t lda_dev = pitch/sizeof(FP_TYPE); cudaMalloc ((void **) &dev_X, sizeof(FP_TYPE) * vlx_dev * th.incx); cudaMalloc ((void **) &dev_Y, sizeof(FP_TYPE) * vly_dev * th.incy); // memcpy from hst to device cublasSetMatrix (rda_dev, cda_dev, sizeof(FP_TYPE), hst_A, lda_hst, dev_A, lda_dev); cublasSetVector (vlx_dev * th.incx, sizeof(FP_TYPE), hst_X, 1, dev_X, 1); // --------------------------------------------- #endif printf ("%d\t%d\t--", th.dim_m_dev, th.dim_n_dev); get_routine_theoretial_performance (&th); // execution --------------------------------- if (th.mode == 'p') { // warm up for (i = 0; i < WLOOP; i++) { #if defined (CUOZBLAS) OzGEMV (&ha, cublas_trans_const_m(th.tranA), rda_dev, cda_dev, &alpha, dev_A, lda_dev, dev_X, 1, &beta, dev_Y, 1); #else OzGEMV (&ha, th.tranA, rda_dev, cda_dev, alpha, hst_A, lda_hst, hst_X, 1, beta, hst_Y, 1); #endif } t0 = gettime (); for (i = 0; i < NLOOP; i++) { #if defined (CUOZBLAS) OzGEMV (&ha, cublas_trans_const_m(th.tranA), rda_dev, cda_dev, &alpha, dev_A, lda_dev, dev_X, 1, &beta, dev_Y, 1); #else OzGEMV (&ha, th.tranA, rda_dev, cda_dev, alpha, hst_A, lda_hst, hst_X, 1, beta, hst_Y, 1); #endif } t1 = gettime (); sec = (t1 - t0) / NLOOP; printf ("\t%1.3e\t%1.3e\t%1.3e\t%d", sec, (th.routine_flops / sec) * 1.0e-9, (th.routine_bytes / sec) * 1.0e-9, NLOOP); } if (th.mode == 'c') { #if defined (CUOZBLAS) cublasSetVector (vly_dev * th.incy, sizeof(FP_TYPE), hst_Y, 1, dev_Y, 1); OzGEMV (&ha, cublas_trans_const_m(th.tranA), rda_dev, cda_dev, &alpha, dev_A, lda_dev, dev_X, 1, &beta, dev_Y, 1); #else mublasCopyMat (vly_dev * th.incy, 1, 0, hst_Y, hst_Y_t); OzGEMV (&ha, th.tranA, rda_dev, cda_dev, alpha, hst_A, lda_hst, hst_X, 1, beta, hst_Y_t, 1); #endif } // ------------------------------------------- #if defined (CUOZBLAS) if (th.mode == 'c') cublasGetVector (vly_dev * th.incy, sizeof(FP_TYPE), dev_Y, 1, hst_Y_t, 1); cudaFree (dev_A); cudaFree (dev_X); cudaFree (dev_Y); #endif if (th.mode == 'c') { mpreal alpha_r = 0; mpreal beta_r = 0; mpreal *hst_A_r = new mpreal[lda_hst * cda_dev]; mpreal *hst_X_r = new mpreal[vlx_dev * th.incx]; mpreal *hst_Y_r = new mpreal[vly_dev * th.incy]; mublasConvMat (1, 1, 0, &alpha, &alpha_r); mublasConvMat (1, 1, 0, &beta, &beta_r); mublasConvMat (rda_dev, cda_dev, lda_hst, hst_A, hst_A_r); mublasConvMat (vlx_dev * th.incx, 1, 0, hst_X, hst_X_r); mublasConvMat (vly_dev * th.incy, 1, 0, hst_Y, hst_Y_r); Rgemv (&th.tranA,rda_dev,cda_dev,alpha_r,hst_A_r,lda_hst,hst_X_r,1,beta_r,hst_Y_r,1); mublasCheckMatrix (vly_dev * th.incy, 1, hst_Y_t, vly_dev * th.incy, hst_Y_r, vly_dev * th.incy); delete[]hst_A_r; delete[]hst_X_r; delete[]hst_Y_r; } #if defined (CUOZBLAS) print_cuozblas_info1 (&th, &ha); #else print_ozblas_info1 (&th, &ha); #endif printf ("\n"); dim_dev_increment (&th); if (th.dim_m_const && th.dim_n_const) break; } // -------------------------------------------- delete[]hst_A; delete[]hst_X; delete[]hst_Y; delete[]hst_Y_t; #if defined (CUOZBLAS) cuozblasDestroy (&ha); #else ozblasDestroy (&ha); #endif // -------------------------------------------- return 0; } ozblas-1.2a17/testing/testing_setting.h0000644000175200017520000000057113725177602020526 0ustar mukunokimukunoki#ifndef TESTING_SETTING #define TESTING_SETTING // ***** SETTING *********************************** #define SIZE_T size_t #define CUBLAS_POINTER_MODE_VALUE CUBLAS_POINTER_MODE_HOST #define WLOOP 3 // (times) #define NLOOP 10 // (times) #define MPFR_PREC 2048 // (bits) #define WORK_MEM_SIZE 10e9 // (bytes) // ***** SETTING *********************************** #endif ozblas-1.2a17/testing/testing_level2s.cpp0000644000175200017520000001770613725177602020770 0ustar mukunokimukunoki#define ROUTINE_FLOPS(m,n,k) (2*k+2*m+m) #define ROUTINE_BYTES(m,n,k,s) ((k+2*m+n)*s+(k+m+1)*4) #include "testing_common.h" #include "testing_common.cpp" void mpfrCsrmv ( testingHandle_t th, int32_t m, int32_t n, int32_t nnz, mpreal alpha, mpreal* matA, int32_t* matAptr, int32_t* matAind, mpreal* x, mpreal beta, mpreal* y ) { int32_t i, j; mpreal t; for(j = 0; j < m; j++) { t = 0.; for(i = matAptr[j]; i < matAptr[j+1]; i++) t = t + matA[i] * x[matAind[i]]; y[j] = alpha * t + beta * y[j]; } } /* FP_TYPE mpfrCsrmvCond ( testingHandle_t th, int32_t m, int32_t n, int32_t nnz, mpreal* matA, int32_t* matAptr, int32_t* matAind, mpreal* x ) { // cond (xTy) := 2|xT||y|/|xTy|, xTy neq 0 int i, j; mpreal y, t, s; mpreal condmax = 0.; mpreal nrmAi, nrmX, condi; for(j = 0; j < m; j++) { y = 0.; // nrmA & nrmX nrmAi = 0.; nrmX = 0.; for(i = matAptr[j]; i < matAptr[j+1]; i++) { t = matA[i]; nrmAi += t*t; s = x[matAind[i]]; nrmX += s*s; } nrmAi = sqrt(nrmAi); nrmX = sqrt(nrmX); // y = Ax for(i = matAptr[j]; i < matAptr[j+1]; i++) y = y + matA[i] * x[matAind[i]]; condi = 2.*nrmAi*nrmX/y; if (condi > condmax) condmax = condi; } return mpfr_get_d(condmax,MPFR_RNDN); } */ int32_t main (int32_t argc, char **argv) { int32_t i; double t0, t1, sec; mpfr_set_default_prec(MPFR_PREC); mpreal::set_default_prec(MPFR_PREC); // testing setup ------------------------------ testingHandle_t th; testingCreate (argc, argv, &th); print_info1 (&th); // -------------------------------------------- // library setup ------------------------------ #if defined (CUOZBLAS) cusparseMatDescr_t descrA; cusparseCreateMatDescr (&descrA); cusparseSetMatType (descrA, CUSPARSE_MATRIX_TYPE_GENERAL); cusparseSetMatIndexBase (descrA, CUSPARSE_INDEX_BASE_ZERO); #else char descrA[4]; descrA[0] = 'G'; descrA[3] = 'C'; #if defined (OZBLAS) && defined (MKL) if (th.useGemmFlag) descrA[3] = 'F'; #endif #endif #if defined (CUOZBLAS) cuozblasHandle_t ha; cuozblasCreate (&ha, WORK_MEM_SIZE, th.numSplitArraysMax, th.fastmode, th.repromode, th.summode); cuozblasProfilingSetting (&ha, th.useGemmFlag, th.useMyGemmFlag, th.useBatchedGemmFlag); #else ozblasHandle_t ha; ozblasCreate (&ha, WORK_MEM_SIZE, th.numSplitArraysMax, th.fastmode, th.repromode, th.summode); ozblasProfilingSetting (&ha, th.useGemmFlag, th.useMyGemmFlag, th.useBatchedGemmFlag); #endif // -------------------------------------------- // memory setup ------------------------------- struct sparse_matrix_t* hst_A__ = load_sparse_matrix (MATRIX_MARKET, th.mtx_file); sparse_matrix_expand_symmetric_storage (hst_A__); int errcode = sparse_matrix_convert (hst_A__, CSR); if (errcode != 0) { fprintf (stderr, "err: conversion failed.\n"); free (hst_A__); exit (1); } FP_TYPE *VAL; struct csr_matrix_t* hst_A_ = (struct csr_matrix_t*) hst_A__->repr; int32_t m = hst_A_->m; int32_t n = hst_A_->n; int32_t nnz = hst_A_->nnz; // malloc host memory FP_TYPE alpha, beta; FP_TYPE *hst_A = new FP_TYPE[nnz]; FP_TYPE *hst_X = new FP_TYPE[n]; FP_TYPE *hst_Y = new FP_TYPE[m]; FP_TYPE *hst_Y_t = new FP_TYPE[m]; int32_t *hst_Colind = hst_A_->colidx; int32_t *hst_Rowptr = hst_A_->rowptr; double *hst_Aptr = (double*) hst_A_->values; for (int32_t i = 0; i < nnz; i++) hst_A[i] = (FP_TYPE) hst_Aptr[i]; // initialize #if defined (OZBLAS) && defined (MKL) // On MKL, SpMM uses one-based indexing while SpMV uses zero-based if (th.useGemmFlag) { for (i = 0; i < m+1; i++) hst_Rowptr[i] += 1; for (i = 0; i < nnz; i++) hst_Colind[i] += 1; } #endif mublasInitMat (1, 1, 0, &alpha, 1., 1); mublasInitMat (1, 1, 0, &beta, 0., 1); mublasInitMat (n, 1, 0, hst_X, th.phi, 3); mublasInitMat (m, 1, 0, hst_Y, th.phi, 3); // -------------------------------------------- print_info2 (&th); // evaluation --------------------------------- #if defined (CUOZBLAS) // malloc device memory FP_TYPE *dev_A, *dev_X, *dev_Y; int32_t *dev_Rowptr, *dev_Colind; cudaMalloc ((void **) &dev_A, sizeof(FP_TYPE) * nnz); cudaMalloc ((void **) &dev_Colind, sizeof(int32_t) * nnz); cudaMalloc ((void **) &dev_Rowptr, sizeof(int32_t) * (m+1)); cudaMalloc ((void **) &dev_X, sizeof (FP_TYPE) * n); cudaMalloc ((void **) &dev_Y, sizeof (FP_TYPE) * m); cudaMemcpy (dev_A, hst_A, sizeof(FP_TYPE) * nnz, cudaMemcpyHostToDevice); cudaMemcpy (dev_Colind, hst_Colind, sizeof(int32_t) * nnz, cudaMemcpyHostToDevice); cudaMemcpy (dev_Rowptr, hst_Rowptr, sizeof(int32_t) * (m+1), cudaMemcpyHostToDevice); cublasSetVector (n, sizeof(FP_TYPE), hst_X, 1, dev_X, 1); cublasSetVector (m, sizeof(FP_TYPE), hst_Y, 1, dev_Y, 1); #endif printf ("%s\t%d\t%d\t%d\t", th.mtx_file, m, n, nnz); th.dim_n_dev = n; th.dim_m_dev = m; th.dim_k_dev = nnz; get_routine_theoretial_performance (&th); // execution --------------------------------- if (th.mode == 'p') { // warm up for (i = 0; i < WLOOP; i++) { #if defined (CUOZBLAS) OzCSRMV (&ha, cusparse_trans_const_m(th.tranA), m, n, nnz, &alpha, descrA, dev_A, dev_Rowptr, dev_Colind, dev_X, &beta, dev_Y); #else OzCSRMV (&ha, th.tranA, m, n, nnz, alpha, descrA, hst_A, hst_Rowptr, hst_Colind, hst_X, beta, hst_Y); #endif } t0 = gettime (); for (i = 0; i < NLOOP; i++) { #if defined (CUOZBLAS) OzCSRMV (&ha, cusparse_trans_const_m(th.tranA), m, n, nnz, &alpha, descrA, dev_A, dev_Rowptr, dev_Colind, dev_X, &beta, dev_Y); #else OzCSRMV (&ha, th.tranA, m, n, nnz, alpha, descrA, hst_A, hst_Rowptr, hst_Colind, hst_X, beta, hst_Y); #endif } t1 = gettime (); sec = (t1 - t0) / NLOOP; printf ("\t%1.3e\t%1.3e\t%1.3e\t%d", sec, (th.routine_flops / sec) * 1.0e-9, (th.routine_bytes / sec) * 1.0e-9, NLOOP); } if (th.mode == 'c') { #if defined (CUOZBLAS) cublasSetVector (m, sizeof(FP_TYPE), hst_Y, 1, dev_Y, 1); OzCSRMV (&ha, cusparse_trans_const_m(th.tranA), m, n, nnz, &alpha, descrA, dev_A, dev_Rowptr, dev_Colind, dev_X, &beta, dev_Y); //// w/o SplitA //FP_TYPE *devASplit; //OzCSRMVSplitA (&ha,cusparse_trans_const_m('n'),m,n,nnz,descrA,dev_A,dev_Rowptr,dev_Colind,&devASplit); //OzCSRMV (&ha, cusparse_trans_const_m(th.tranA), m, n, nnz, &alpha, descrA, devASplit, dev_Rowptr, dev_Colind, dev_X, &beta, dev_Y); #else mublasCopyMat (m, 1, 0, hst_Y, hst_Y_t); OzCSRMV (&ha, th.tranA, m, n, nnz, alpha, descrA, hst_A, hst_Rowptr, hst_Colind, hst_X, beta, hst_Y_t); #endif } // ------------------------------------------- #if defined (CUOZBLAS) if (th.mode == 'c') cublasGetVector (m, sizeof(FP_TYPE), dev_Y, 1, hst_Y_t, 1); cudaFree (dev_A); cudaFree (dev_Colind); cudaFree (dev_Rowptr); cudaFree (dev_X); cudaFree (dev_Y); #endif if (th.mode == 'c') { mpreal alpha_r = 0; mpreal beta_r = 0; mpreal *hst_A_r = new mpreal[nnz]; mpreal *hst_X_r = new mpreal[n]; mpreal *hst_Y_r = new mpreal[m]; mublasConvMat (1, 1, 0, &alpha, &alpha_r); mublasConvMat (1, 1, 0, &beta, &beta_r); mublasConvMat (nnz, 1, 0, hst_A, hst_A_r); mublasConvMat (n, 1, 0, hst_X, hst_X_r); mublasConvMat (m, 1, 0, hst_Y, hst_Y_r); #if defined (OZBLAS) && defined (MKL) if (th.useGemmFlag) { for (i = 0; i < m+1; i++) hst_Rowptr[i] -= 1; for (i = 0; i < nnz; i++) hst_Colind[i] -= 1; } #endif mpfrCsrmv (th, m, n, nnz, alpha_r, hst_A_r, hst_Rowptr, hst_Colind, hst_X_r, beta_r, hst_Y_r); // FP_TYPE condmax; // condmax = mpfrCsrmvCond (th,m, n, nnz, hst_A_r, hst_Rowptr, hst_Colind, hst_X_r); // printf ("### condmax = %1.3e\n", condmax); mublasCheckMatrix (m, 1, hst_Y_t, m, hst_Y_r, m); delete[]hst_A_r; delete[]hst_X_r; delete[]hst_Y_r; } #if defined (CUOZBLAS) print_cuozblas_info1 (&th, &ha); #else print_ozblas_info1 (&th, &ha); #endif printf ("\n"); // -------------------------------------------- // shutdown ----------------------------------- delete[]hst_A; delete[]hst_X; delete[]hst_Y; delete[]hst_Y_t; #if defined (CUOZBLAS) cuozblasDestroy (&ha); #else ozblasDestroy (&ha); #endif // -------------------------------------------- destroy_sparse_matrix(hst_A__); return 0; } ozblas-1.2a17/testing/testing_solver.cpp0000644000175200017520000001026113725177602020713 0ustar mukunokimukunoki#include "testing_common.h" #include "testing_common.cpp" int32_t main (int32_t argc, char **argv) { // testing setup ------------------------------ testingHandle_t th; testingCreate (argc, argv, &th); #if defined (OZBLAS) // On MKL, CSRMM is slow th.useGemmFlag = 0; #endif print_info1 (&th); #if defined (OZBLAS) printf ("## Note: force useGemmFlag = 0\n"); #endif // -------------------------------------------- // library setup ------------------------------ #if defined (CUOZBLAS) cusparseMatDescr_t descrA; cusparseCreateMatDescr (&descrA); cusparseSetMatType (descrA, CUSPARSE_MATRIX_TYPE_GENERAL); cusparseSetMatIndexBase (descrA, CUSPARSE_INDEX_BASE_ZERO); #else char descrA[4]; descrA[0] = 'G'; descrA[3] = 'C'; #if defined (OZBLAS) && defined (MKL) if (th.useGemmFlag) descrA[3] = 'F'; #endif #endif #if defined (CUOZBLAS) cuozblasHandle_t ha; cuozblasCreate (&ha, WORK_MEM_SIZE, th.numSplitArraysMax, th.fastmode, th.repromode, th.summode); cuozblasProfilingSetting (&ha, th.useGemmFlag, th.useMyGemmFlag, th.useBatchedGemmFlag); #else ozblasHandle_t ha; ozblasCreate (&ha, WORK_MEM_SIZE, th.numSplitArraysMax, th.fastmode, th.repromode, th.summode); ozblasProfilingSetting (&ha, th.useGemmFlag, th.useMyGemmFlag, th.useBatchedGemmFlag); #endif // -------------------------------------------- // memory setup ------------------------------- struct sparse_matrix_t* hst_A__ = load_sparse_matrix (MATRIX_MARKET, th.mtx_file); sparse_matrix_expand_symmetric_storage (hst_A__); int errcode = sparse_matrix_convert (hst_A__, CSR); if (errcode != 0) { fprintf (stderr, "err: conversion failed.\n"); free (hst_A__); exit (1); } struct csr_matrix_t* hst_A_ = (struct csr_matrix_t*) hst_A__->repr; int32_t m = hst_A_->m; int32_t n = hst_A_->n; int32_t nnz = hst_A_->nnz; // malloc host memory FP_TYPE alpha, beta; FP_TYPE *hst_A = new FP_TYPE[nnz]; FP_TYPE *hst_X = new FP_TYPE[n]; FP_TYPE *hst_B = new FP_TYPE[n]; int32_t *hst_Colind = hst_A_->colidx; int32_t *hst_Rowptr = hst_A_->rowptr; double *hst_Aptr = (double*) hst_A_->values; for (int32_t i = 0; i < nnz; i++) hst_A[i] = (FP_TYPE) hst_Aptr[i]; #if defined (OZBLAS) && defined (MKL) // On MKL, SpMM uses one-based indexing while SpMV uses zero-based if (th.useGemmFlag) { for (int32_t i = 0; i < m+1; i++) hst_Rowptr[i] += 1; for (int32_t i = 0; i < nnz; i++) hst_Colind[i] += 1; } #endif int32_t maxiter = th.maxiter; double tol = 1e-16; ha.trueres = th.trueres; ha.verbose = th.verbose; // initialize mublasInitMat (n, 1, 0, hst_X, 1., 1); mublasInitMat (n, 1, 0, hst_B, 1., 1); // -------------------------------------------- print_info2 (&th); // evaluation --------------------------------- #if defined (CUOZBLAS) // malloc device memory FP_TYPE *dev_A, *dev_X, *dev_B; int32_t *dev_Rowptr, *dev_Colind; cudaMalloc ((void **) &dev_A, sizeof (FP_TYPE) * nnz); cudaMalloc ((void **) &dev_X, sizeof (FP_TYPE) * n); cudaMalloc ((void **) &dev_B, sizeof (FP_TYPE) * n); cudaMalloc ((void **) &dev_Colind, sizeof (int32_t) * nnz); cudaMalloc ((void **) &dev_Rowptr, sizeof (int32_t) * (m+1)); cudaMemcpy (dev_A, hst_A, sizeof(FP_TYPE) * nnz, cudaMemcpyHostToDevice); cudaMemcpy (dev_Colind, hst_Colind, sizeof(int32_t) * nnz, cudaMemcpyHostToDevice); cudaMemcpy (dev_Rowptr, hst_Rowptr, sizeof(int32_t) * (m+1), cudaMemcpyHostToDevice); cublasSetVector (n, sizeof(FP_TYPE), hst_X, 1, dev_X, 1); cublasSetVector (n, sizeof(FP_TYPE), hst_B, 1, dev_B, 1); #endif printf ("## %s\t%d\t%d\t%d\n", th.mtx_file, m, n, nnz); #if defined (CUOZBLAS) OzCG (&ha, cusparse_trans_const_m(th.tranA), n, nnz, descrA, dev_A, dev_Rowptr, dev_Colind, dev_B, dev_X, maxiter, tol); #else OzCG (&ha, th.tranA, n, nnz, descrA, hst_A, hst_Rowptr, hst_Colind, hst_B, hst_X, maxiter, tol); #endif // shutdown ----------------------------------- #if defined (CUOZBLAS) cudaFree (dev_A); cudaFree (dev_Rowptr); cudaFree (dev_Colind); cudaFree (dev_X); cudaFree (dev_B); #endif delete[]hst_A; delete[]hst_X; delete[]hst_B; #if defined (CUOZBLAS) cuozblasDestroy (&ha); #else ozblasDestroy (&ha); #endif destroy_sparse_matrix(hst_A__); // -------------------------------------------- return 0; } ozblas-1.2a17/testing/testing_level1b.cpp0000644000175200017520000001075013725177602020736 0ustar mukunokimukunoki#define ROUTINE_FLOPS(m,n,k) (2*n) #define ROUTINE_BYTES(m,n,k,s) (2*n*s) #include "testing_common.h" #include "testing_common.cpp" int32_t main (int32_t argc, char **argv) { int32_t i; double t0, t1, sec; mpreal hst_result_r; mpfr_set_default_prec(MPFR_PREC); mpreal::set_default_prec(MPFR_PREC); // testing setup ------------------------------ testingHandle_t th; testingCreate (argc, argv, &th); print_info1 (&th); // -------------------------------------------- // library setup ------------------------------ #if defined (CUOZBLAS) cuozblasHandle_t ha; cuozblasCreate (&ha, WORK_MEM_SIZE, th.numSplitArraysMax, th.fastmode, th.repromode, th.summode); cuozblasProfilingSetting (&ha, th.useGemmFlag, th.useMyGemmFlag, th.useBatchedGemmFlag); #else ozblasHandle_t ha; ozblasCreate (&ha, WORK_MEM_SIZE, th.numSplitArraysMax, th.fastmode, th.repromode, th.summode); ozblasProfilingSetting (&ha, th.useGemmFlag, th.useMyGemmFlag, th.useBatchedGemmFlag); #endif // -------------------------------------------- // memory setup ------------------------------- FP_TYPE hst_result_t; #if defined (CUOZBLAS) FP_TYPE *dev_X, *dev_Y;//, *ptr_result_t; /* if (th.pmode) cudaMalloc ((void **) &ptr_result_t, sizeof(FP_TYPE)); else ptr_result_t = &hst_result_t; if (th.pmode) cudaMalloc ((void **) &ptr_result_r, sizeof(FP_TYPE)); else ptr_result_r = &hst_result_r; */ #endif // dim setup dim_hst_setup (&th); // malloc host memory FP_TYPE *hst_X = new FP_TYPE[th.dim_n_hst * th.incx]; FP_TYPE *hst_Y = new FP_TYPE[th.dim_n_hst * th.incy]; // initialize (1:val, 2:phi, 3:erange) mublasInitMat (th.dim_n_hst*th.incx, 1, 0, hst_X, th.phi, 3); mublasInitMat (th.dim_n_hst*th.incy, 1, 0, hst_Y, th.phi, 3); // -------------------------------------------- print_info2 (&th); // evaluation --------------------------------- dim_dev_setup (&th); while (1) { if (th.dim_n_const == 0 && th.dim_n_dev > th.dim_stop) break; #if defined (CUOZBLAS) cudaMalloc ((void **) &dev_X, sizeof(FP_TYPE) * th.dim_n_dev * th.incx); cudaMalloc ((void **) &dev_Y, sizeof(FP_TYPE) * th.dim_n_dev * th.incy); cublasSetVector(th.dim_n_dev * th.incx, sizeof(FP_TYPE), hst_X, 1, dev_X, 1); cublasSetVector(th.dim_n_dev * th.incy, sizeof(FP_TYPE), hst_Y, 1, dev_Y, 1); #endif printf ("--\t%d\t--", th.dim_n_dev); get_routine_theoretial_performance (&th); // execution --------------------------------- if (th.mode == 'p') { // warm up for (i = 0; i < WLOOP; i++) { #if defined (CUOZBLAS) OzDOT (&ha, th.dim_n_dev, dev_X, 1, dev_Y, 1, &hst_result_t); #else hst_result_t = OzDOT (&ha, th.dim_n_dev, hst_X, 1, hst_Y, 1); #endif } t0 = gettime (); for (i = 0; i < NLOOP; i++) { #if defined (CUOZBLAS) OzDOT (&ha, th.dim_n_dev, dev_X, 1, dev_Y, 1, &hst_result_t); #else hst_result_t = OzDOT (&ha, th.dim_n_dev, hst_X, 1, hst_Y, 1); #endif } t1 = gettime (); sec = (t1 - t0) / NLOOP; printf ("\t%1.3e\t%1.3e\t%1.3e\t%d", sec, (th.routine_flops / sec) * 1.0e-9, (th.routine_bytes / sec) * 1.0e-9, NLOOP); } if (th.mode == 'c') { #if defined (CUOZBLAS) OzDOT (&ha, th.dim_n_dev, dev_X, 1, dev_Y, 1, &hst_result_t); #else hst_result_t = OzDOT (&ha, th.dim_n_dev, hst_X, 1, hst_Y, 1); #endif } // ------------------------------------------- #if defined (CUOZBLAS) // if (th.mode == 'c') // if (th.pmode) cublasGetVector (1, sizeof(FP_TYPE), ptr_result_t, 1, &hst_result_t, 1); cudaFree (dev_X); cudaFree (dev_Y); #endif if (th.mode == 'c') { mpreal *hst_X_r = new mpreal[th.dim_n_hst]; mpreal *hst_Y_r = new mpreal[th.dim_n_hst]; mublasConvMat (th.dim_n_hst*th.incx, 1, 0, hst_X, hst_X_r); mublasConvMat (th.dim_n_hst*th.incy, 1, 0, hst_Y, hst_Y_r); hst_result_r = Rdot (th.dim_n_dev,hst_X_r,1,hst_Y_r,1); mublasCheckMatrix (1, 1, &hst_result_t, 1, &hst_result_r, 1); delete[]hst_X_r; delete[]hst_Y_r; } #if defined (CUOZBLAS) print_cuozblas_info1 (&th, &ha); #else print_ozblas_info1 (&th, &ha); #endif printf ("\n"); dim_dev_increment (&th); if (th.dim_m_const && th.dim_n_const && th.dim_k_const) break; } // -------------------------------------------- // shutdown ----------------------------------- delete[]hst_X; delete[]hst_Y; /* #if defined (CUOZBLAS) if (th.pmode) cudaFree (ptr_result_r); if (th.pmode) cudaFree (ptr_result_t); #endif */ #if defined (CUOZBLAS) cuozblasDestroy (&ha); #else ozblasDestroy (&ha); #endif // -------------------------------------------- return 0; } ozblas-1.2a17/testing/testing_common.h0000644000175200017520000001071613725177602020343 0ustar mukunokimukunoki#ifndef TESTING_COMMON #define TESTING_COMMON // header ------------------------------------------ #include "testing_setting.h" #include #include #include #include #include #define __STDC_FORMAT_MACROS #include #include #include #include #include #include #if defined (CUDA) #include #include #include #include #endif #if defined (CUOZBLAS) #include #endif #if defined (OZBLAS) #include #endif #include #include #if defined (CSRMV) || defined (CG) extern "C" { #include #include #include } #endif //#define size_t uint32_t #if defined (PREC_S) #define FP_TYPE float #define MAGMA_OPTS magma_sopts #define MAGMA_MATRIX magma_s_matrix #define MAGMA_CSR_MTX magma_s_csr_mtx #define MAGMA_MTRANSFER magma_smtransfer #define MAGMA_MFREE magma_smfree //#define MAGMA_COPY magma_scopy //#define MAGMA_VINIT magma_svinit #define MAGMA_SCAL magma_sscal //#define MAGMA_ZERO MAGMA_S_ZERO //#define MAGMA_ONE MAGMA_S_ONE //#define MAGMA_MAKE MAGMA_S_MAKE //#define MAGMA_SOLVER magma_s_solver //#define MAGMA_SOLVER_PAR magma_s_solver_par //#define MAGMA_SOLVERINFO magma_ssolverinfo //#define MAGMA_SOLVERINFO_INIT magma_ssolverinfo_init //#define MAGMA_RESIDUALVEC magma_sresidualvec //#define MAGMA_PARSE_OPTS magma_sparse_opts #if defined (CUOZBLAS) #define OzCG cuozblasScg #define OzDOT cuozblasSdot #define OzAXPY cuozblasSaxpy #define OzGEMV cuozblasSgemv #define OzGEMM cuozblasSgemm #define OzNRM2 cuozblasSnrm2 #define OzCSRMV cuozblasScsrmv #define OzCSRMVSplitA cuozblasScsrmvSplitA #elif defined (OZBLAS) #define OzCG ozblasScg #define OzDOT ozblasSdot #define OzAXPY ozblasSaxpy #define OzGEMV ozblasSgemv #define OzGEMM ozblasSgemm #define OzNRM2 ozblasSnrm2 #define OzCSRMV ozblasScsrmv #define OzCSRMVSplitA ozblasScsrmvSplitA #endif #elif defined (PREC_D) #define FP_TYPE double #define MAGMA_OPTS magma_dopts #define MAGMA_MATRIX magma_d_matrix #define MAGMA_CSR_MTX magma_d_csr_mtx #define MAGMA_MTRANSFER magma_dmtransfer #define MAGMA_MFREE magma_dmfree #define MAGMA_COPY magma_dcopy #define MAGMA_VINIT magma_dvinit #define MAGMA_SCAL magma_dscal #define MAGMA_ZERO MAGMA_D_ZERO #define MAGMA_ONE MAGMA_D_ONE #define MAGMA_MAKE MAGMA_D_MAKE #define MAGMA_SOLVER magma_d_solver #define MAGMA_SOLVER_PAR magma_d_solver_par #define MAGMA_SOLVERINFO magma_dsolverinfo #define MAGMA_SOLVERINFO_INIT magma_dsolverinfo_init #define MAGMA_RESIDUALVEC magma_dresidualvec #define MAGMA_PARSE_OPTS magma_dparse_opts #if defined (CUOZBLAS) #define OzCG cuozblasDcg #define OzDOT cuozblasDdot #define OzAXPY cuozblasDaxpy #define OzGEMV cuozblasDgemv #define OzGEMM cuozblasDgemm #define OzNRM2 cuozblasDnrm2 #define OzCSRMV cuozblasDcsrmv #define OzCSRMVSplitA cuozblasDcsrmvSplitA #elif defined (OZBLAS) #define OzCG ozblasDcg #define OzDOT ozblasDdot #define OzAXPY ozblasDaxpy #define OzGEMV ozblasDgemv #define OzGEMM ozblasDgemm #define OzNRM2 ozblasDnrm2 #define OzCSRMV ozblasDcsrmv #define OzCSRMVSplitA ozblasDcsrmvSplitA #endif #elif defined (PREC_SD) #define FP_TYPE float #define MAGMA_OPTS magma_sopts #define MAGMA_MATRIX magma_s_matrix #define MAGMA_CSR_MTX magma_s_csr_mtx #define MAGMA_MTRANSFER magma_smtransfer #define MAGMA_MFREE magma_smfree #define MAGMA_COPY magma_scopy #define MAGMA_VINIT magma_svinit #define MAGMA_SCAL magma_sscal #define MAGMA_ZERO MAGMA_S_ZERO #define MAGMA_ONE MAGMA_S_ONE #define MAGMA_MAKE MAGMA_S_MAKE #define MAGMA_SOLVER magma_s_solver #define MAGMA_SOLVER_PAR magma_s_solver_par #define MAGMA_SOLVERINFO magma_ssolverinfo #define MAGMA_SOLVERINFO_INIT magma_ssolverinfo_init #define MAGMA_RESIDUALVEC magma_sresidualvec #define MAGMA_PARSE_OPTS magma_sparse_opts #if defined (CUOZBLAS) #define OzCG cuozblasSDcg #define OzDOT cuozblasSDdot #define OzAXPY cuozblasSDaxpy #define OzGEMV cuozblasSDgemv #define OzGEMM cuozblasSDgemm #define OzNRM2 cuozblasSDnrm2 #define OzCSRMV cuozblasSDcsrmv #define OzCSRMVSplitA cuozblasSDcsrmvSplitA #elif defined (OZBLAS) #define OzCG ozblasSDcg #define OzDOT ozblasSDdot #define OzAXPY ozblasSDaxpy #define OzGEMV ozblasSDgemv #define OzGEMM ozblasSDgemm #define OzNRM2 ozblasSDnrm2 #define OzCSRMV ozblasSDcsrmv #define OzCSRMVSplitA ozblasSDcsrmvSplitA #endif #endif #endif ozblas-1.2a17/testing/testing_level3.cpp0000644000175200017520000001635313725177602020603 0ustar mukunokimukunoki#define ROUTINE_FLOPS(m,n,k) (2*m*n*k+3*m*n) #define ROUTINE_BYTES(m,n,k,s) ((m*k+k*n+2*m*n)*s) #include "testing_common.h" #include "testing_common.cpp" int32_t main (int32_t argc, char **argv) { int32_t i; int32_t lda_dev, lda_hst, rda_dev, rda_hst, cda_dev, cda_hst; int32_t ldb_dev, ldb_hst, rdb_dev, rdb_hst, cdb_dev, cdb_hst; int32_t ldc_dev, ldc_hst, rdc_dev, rdc_hst, cdc_dev, cdc_hst; double t0, t1, sec; mpfr_set_default_prec(MPFR_PREC); mpreal::set_default_prec(MPFR_PREC); // testing setup ------------------------------ testingHandle_t th; testingCreate (argc, argv, &th); print_info1 (&th); // library setup ------------------------------ #if defined (CUOZBLAS) cuozblasHandle_t ha; cuozblasCreate (&ha, WORK_MEM_SIZE, th.numSplitArraysMax, th.fastmode, th.repromode, th.summode); cuozblasProfilingSetting (&ha, th.useGemmFlag, th.useMyGemmFlag, th.useBatchedGemmFlag); cuozblasProfilingSetting2 (&ha, th.sparseThreshold); #else ozblasHandle_t ha; ozblasCreate (&ha, WORK_MEM_SIZE, th.numSplitArraysMax, th.fastmode, th.repromode, th.summode); ozblasProfilingSetting (&ha, th.useGemmFlag, th.useMyGemmFlag, th.useBatchedGemmFlag); #endif // -------------------------------------------- // memory setup ------------------------------- // dim setup dim_hst_setup (&th); // A rda_hst = th.dim_stop; cda_hst = th.dim_stop; if (th.tranA == 'N' || th.tranA == 'n') { if (th.dim_m_const > 0) rda_hst = th.dim_m_const; if (th.dim_k_const > 0) cda_hst = th.dim_k_const; } else { if (th.dim_k_const > 0) rda_hst = th.dim_k_const; if (th.dim_m_const > 0) cda_hst = th.dim_m_const; } lda_hst = rda_hst; // B rdb_hst = th.dim_stop; cdb_hst = th.dim_stop; if (th.tranB == 'N' || th.tranB == 'n') { if (th.dim_k_const > 0) rdb_hst = th.dim_k_const; if (th.dim_n_const > 0) cdb_hst = th.dim_n_const; } else { if (th.dim_n_const > 0) rdb_hst = th.dim_n_const; if (th.dim_k_const > 0) cdb_hst = th.dim_k_const; } ldb_hst = rdb_hst; // C rdc_hst = th.dim_stop; cdc_hst = th.dim_stop; if (th.dim_m_const > 0) rdc_hst = th.dim_m_const; if (th.dim_n_const > 0) cdc_hst = th.dim_n_const; ldc_hst = rdc_hst; // malloc host memory FP_TYPE alpha, beta; FP_TYPE *hst_A = new FP_TYPE[lda_hst * cda_hst]; FP_TYPE *hst_B = new FP_TYPE[ldb_hst * cdb_hst]; FP_TYPE *hst_C = new FP_TYPE[ldc_hst * cdc_hst]; FP_TYPE *hst_C_t = new FP_TYPE[ldc_hst * cdc_hst]; // initialize (1:val, 2:phi, 3:erange) mublasInitMat (1, 1, 0, &alpha, 1., 1); mublasInitMat (1, 1, 0, &beta, 0., 1); mublasInitMat (lda_hst, cda_hst, lda_hst, hst_A, th.phi, 3); mublasInitMat (ldb_hst, cdb_hst, ldb_hst, hst_B, th.phi, 3); mublasInitMat (ldc_hst, cdc_hst, ldc_hst, hst_C, 0., 1); // -------------------------------------------- print_info2 (&th); // evaluation --------------------------------- dim_dev_setup (&th); while (1) { if ((th.dim_m_const == 0 && th.dim_m_dev > th.dim_stop) || (th.dim_n_const == 0 && th.dim_n_dev > th.dim_stop) || (th.dim_k_const == 0 && th.dim_k_dev > th.dim_stop)) break; // dim setup // A if (th.tranA == 'N' || th.tranA == 'n') { rda_dev = th.dim_m_dev; cda_dev = th.dim_k_dev; } else { rda_dev = th.dim_k_dev; cda_dev = th.dim_m_dev; } // B if (th.tranB == 'N' || th.tranB == 'n') { rdb_dev = th.dim_k_dev; cdb_dev = th.dim_n_dev; } else { rdb_dev = th.dim_n_dev; cdb_dev = th.dim_k_dev; } // C rdc_dev = th.dim_m_dev; cdc_dev = th.dim_n_dev; #if defined (CUOZBLAS) // malloc device memory FP_TYPE *dev_A, *dev_B, *dev_C; size_t pitch; cudaMallocPitch ((void **) &dev_A, &pitch, sizeof(FP_TYPE) * rda_dev, cda_dev); lda_dev = pitch/sizeof(FP_TYPE); cudaMallocPitch ((void **) &dev_B, &pitch, sizeof(FP_TYPE) * rdb_dev, cdb_dev); ldb_dev = pitch/sizeof(FP_TYPE); cudaMallocPitch ((void **) &dev_C, &pitch, sizeof(FP_TYPE) * rdc_dev, cdc_dev); ldc_dev = pitch/sizeof(FP_TYPE); // memcpy from host to device cublasSetMatrix(rda_dev, cda_dev, sizeof(FP_TYPE), hst_A, lda_hst, dev_A, lda_dev); cublasSetMatrix(rdb_dev, cdb_dev, sizeof(FP_TYPE), hst_B, ldb_hst, dev_B, ldb_dev); #endif printf ("%d\t%d\t%d", th.dim_m_dev, th.dim_n_dev, th.dim_k_dev); get_routine_theoretial_performance (&th); // execution --------------------------------- if (th.mode == 'p') { // warm up for (i = 0; i < WLOOP; i++) { #if defined (CUOZBLAS) OzGEMM (&ha, cublas_trans_const_m(th.tranA), cublas_trans_const_m(th.tranB), th.dim_m_dev, th.dim_n_dev, th.dim_k_dev, &alpha, dev_A, lda_dev, dev_B, ldb_dev, &beta, dev_C, ldc_dev); #else OzGEMM (&ha, th.tranA, th.tranB, th.dim_m_dev, th.dim_n_dev, th.dim_k_dev, alpha, hst_A, lda_hst, hst_B, ldb_hst, beta, hst_C, ldc_hst); #endif } t0 = gettime (); for (i = 0; i < NLOOP; i++) { #if defined (CUOZBLAS) OzGEMM (&ha, cublas_trans_const_m(th.tranA), cublas_trans_const_m(th.tranB), th.dim_m_dev, th.dim_n_dev, th.dim_k_dev, &alpha, dev_A, lda_dev, dev_B, ldb_dev, &beta, dev_C, ldc_dev); #else OzGEMM (&ha, th.tranA, th.tranB, th.dim_m_dev, th.dim_n_dev, th.dim_k_dev, alpha, hst_A, lda_hst, hst_B, ldb_hst, beta, hst_C, ldc_hst); #endif } t1 = gettime (); sec = (t1 - t0) / NLOOP; printf ("\t%1.3e\t%1.3e\t%1.3e\t%d", sec, (th.routine_flops / sec) * 1.0e-9, (th.routine_bytes / sec) * 1.0e-9, NLOOP); } if (th.mode == 'c') { #if defined (CUOZBLAS) cublasSetMatrix(rdc_dev, cdc_dev, sizeof(FP_TYPE), hst_C, ldc_hst, dev_C, ldc_dev); OzGEMM (&ha, cublas_trans_const_m(th.tranA), cublas_trans_const_m(th.tranB), th.dim_m_dev, th.dim_n_dev, th.dim_k_dev, &alpha, dev_A, lda_dev, dev_B, ldb_dev, &beta, dev_C, ldc_dev); #else mublasCopyMat (rdc_dev, cdc_dev, ldc_hst, hst_C, hst_C_t); OzGEMM (&ha, th.tranA, th.tranB, th.dim_m_dev, th.dim_n_dev, th.dim_k_dev, alpha, hst_A, lda_hst, hst_B, ldb_hst, beta, hst_C_t, ldc_hst); #endif } // ------------------------------------------- #if defined (CUOZBLAS) if (th.mode == 'c') cublasGetMatrix (rdc_dev, cdc_dev, sizeof(FP_TYPE), dev_C, ldc_dev, hst_C_t, ldc_hst); cudaFree (dev_A); cudaFree (dev_B); cudaFree (dev_C); #endif if (th.mode == 'c') { mpreal alpha_r = 0; mpreal beta_r = 0; mpreal *hst_A_r = new mpreal[lda_hst * cda_hst]; mpreal *hst_B_r = new mpreal[ldb_hst * cdb_hst]; mpreal *hst_C_r = new mpreal[ldc_hst * cdc_hst]; mublasConvMat (1, 1, 0, &alpha, &alpha_r); mublasConvMat (1, 1, 0, &beta, &beta_r); mublasConvMat (rda_dev, cda_dev, lda_hst, hst_A, hst_A_r); mublasConvMat (rdb_dev, cdb_dev, ldb_hst, hst_B, hst_B_r); mublasConvMat (rdc_dev, cdc_dev, ldc_hst, hst_C, hst_C_r); Rgemm (&th.tranA,&th.tranB,th.dim_m_dev,th.dim_n_dev,th.dim_k_dev,alpha_r,hst_A_r,lda_hst,hst_B_r,ldb_hst,beta_r,hst_C_r,ldc_hst); mublasCheckMatrix (th.dim_m_dev, th.dim_n_dev, hst_C_t, ldc_hst, hst_C_r, ldc_hst); delete[]hst_A_r; delete[]hst_B_r; delete[]hst_C_r; } #if defined (CUOZBLAS) print_cuozblas_info1 (&th, &ha); #else print_ozblas_info1 (&th, &ha); #endif printf ("\n"); dim_dev_increment (&th); if (th.dim_m_const && th.dim_n_const && th.dim_k_const) break; } // -------------------------------------------- delete[]hst_A; delete[]hst_B; delete[]hst_C; delete[]hst_C_t; #if defined (CUOZBLAS) cuozblasDestroy (&ha); #else ozblasDestroy (&ha); #endif return 0; } ozblas-1.2a17/testing/Makefile0000644000175200017520000001651613725177602016611 0ustar mukunokimukunoki# Makefile include ../make.inc # common ------------------------------------- FLAGS = -O3 -Wall -MD # DOT --------------------------------------- ifeq ($(DOT),yes) ifeq ($(PREC_S),yes) CUOZBLAS_OBJS += testing_cuozblas_sdot OZBLAS_OBJS += testing_ozblas_sdot endif ifeq ($(PREC_D),yes) CUOZBLAS_OBJS += testing_cuozblas_ddot OZBLAS_OBJS += testing_ozblas_ddot endif ifeq ($(PREC_SD),yes) CUOZBLAS_OBJS += testing_cuozblas_sddot OZBLAS_OBJS += testing_ozblas_sddot endif endif # GEMV --------------------------------------- ifeq ($(GEMV),yes) ifeq ($(PREC_S),yes) CUOZBLAS_OBJS += testing_cuozblas_sgemv OZBLAS_OBJS += testing_ozblas_sgemv endif ifeq ($(PREC_D),yes) CUOZBLAS_OBJS += testing_cuozblas_dgemv OZBLAS_OBJS += testing_ozblas_dgemv endif ifeq ($(PREC_SD),yes) #CUOZBLAS_OBJS += testing_cuozblas_sdgemv #OZBLAS_OBJS += testing_ozblas_sdgemv endif endif # CSRMV --------------------------------------- ifeq ($(CSRMV),yes) ifeq ($(PREC_S),yes) CUOZBLAS_OBJS += testing_cuozblas_scsrmv ifeq ($(CPU_BLAS),MKL) OZBLAS_OBJS += testing_ozblas_scsrmv endif endif ifeq ($(PREC_D),yes) CUOZBLAS_OBJS += testing_cuozblas_dcsrmv ifeq ($(CPU_BLAS),MKL) OZBLAS_OBJS += testing_ozblas_dcsrmv endif endif ifeq ($(PREC_SD),yes) CUOZBLAS_OBJS += testing_cuozblas_sdcsrmv ifeq ($(CPU_BLAS),MKL) OZBLAS_OBJS += testing_ozblas_sdcsrmv endif endif endif # GEMM --------------------------------------- ifeq ($(GEMM),yes) ifeq ($(PREC_S),yes) CUOZBLAS_OBJS += testing_cuozblas_sgemm OZBLAS_OBJS += testing_ozblas_sgemm endif ifeq ($(PREC_D),yes) CUOZBLAS_OBJS += testing_cuozblas_dgemm OZBLAS_OBJS += testing_ozblas_dgemm endif ifeq ($(PREC_SD),yes) #CUOZBLAS_OBJS += testing_cuozblas_sdgemm #OZBLAS_OBJS += testing_ozblas_sdgemm endif endif # CG ----------------------------------------- ifeq ($(CG),yes) ifeq ($(PREC_S),yes) CUOZBLAS_OBJS += testing_cuozblas_scg OZBLAS_OBJS += testing_ozblas_scg endif ifeq ($(PREC_D),yes) CUOZBLAS_OBJS += testing_cuozblas_dcg OZBLAS_OBJS += testing_ozblas_dcg endif ifeq ($(PREC_SD),yes) CUOZBLAS_OBJS += testing_cuozblas_sdcg OZBLAS_OBJS += testing_ozblas_dcg endif endif # -------------------------------------------- ifeq ($(CUDA),yes) OBJS += $(CUOZBLAS_OBJS) INCS += -I$(CUDA_PATH)/include LDFLAGS += -L$(CUDA_PATH)/lib64 -DCUDA LIBS += -lcudart -lcublas -lcusparse -lcuozblas endif ifeq ($(CPU),yes) OBJS += $(OZBLAS_OBJS) INCS += LDFLAGS += -lozblas ifeq ($(CPU_BLAS),MKL) ifeq ($(CXX),icpc) FLAGS += -qopenmp -lmkl_intel_lp64 -lmkl_core -lmkl_intel_thread -lpthread -limf -DMKL else # gcc with MKL does not work with OMP (maybe linkage issue?) FLAGS += -fopenmp -lmkl_intel_lp64 -lmkl_core -lmkl_sequential -lpthread -DMKL endif endif ifeq ($(CPU_BLAS),OpenBLAS) FLAGS += -lopenblas -DOpenBLAS endif ifeq ($(NVBLAS),yes) FLAGS += -lnvblas -DNVBLAS endif endif INCS += $(MPFR_INC) $(MPACK_INC) -I../include LDFLAGS += $(MPFR_LDFLAGS) $(MPACK_LDFLAGS) -L$(MPACK_DIR)/external/i/MPC/lib/ -L$(MPACK_DIR)/external/i/MPFR/lib/ -L$(MPACK_DIR)/external/i/GMP/lib/ -L../lib -DProfiling ifeq ($(SYSTEM),OFP) LIBS += -Wl,-Bstatic,-lmblas_mpfr,-lmpfrcxx,-lmpfr -fopenmp -lstdc++ -lmpc -lgmp else LIBS += -lmpc -lgmp -lmblas_mpfr -lmpfr -fopenmp -lmpfrcxx -lstdc++ endif ifeq ($(CSRMV),yes) ifeq ($(SYSTEM),OFP) LIBS += -Wl,-Bstatic,-lsparse_matrix_converter,-lbebop_util else LIBS += -lsparse_matrix_converter -lbebop_util endif endif ifeq ($(CG),yes) ifeq ($(SYSTEM),OFP) LIBS += -Wl,-Bstatic,-lsparse_matrix_converter,-lbebop_util else LIBS += -lsparse_matrix_converter -lbebop_util endif endif default: $(OBJS) ifeq ($(CPU),yes) $(OBJS): ../lib/libozblas.a endif ifeq ($(CUDA),yes) $(OBJS): ../lib/libcuozblas.a endif SRC_L1B = testing_level1b.cpp SRC_L2 = testing_level2.cpp SRC_L2S = testing_level2s.cpp SRC_L3 = testing_level3.cpp SRC_SS = testing_solver.cpp ### CUOZBLAS ################################################################################################### testing_cuozblas_sgemm: $(SRC_L3) $(CXX) $< -o $@ $(FLAGS) $(INCS) $(LDFLAGS) $(LIBS) -DCUOZBLAS -DGEMM -DPREC_S testing_cuozblas_sgemv: $(SRC_L2) $(CXX) $< -o $@ $(FLAGS) $(INCS) $(LDFLAGS) $(LIBS) -DCUOZBLAS -DGEMV -DPREC_S testing_cuozblas_scsrmv: $(SRC_L2S) $(CXX) $< -o $@ $(FLAGS) $(INCS) $(LDFLAGS) $(LIBS) -DCUOZBLAS -DCSRMV -DPREC_S testing_cuozblas_sdot: $(SRC_L1B) $(CXX) $< -o $@ $(FLAGS) $(INCS) $(LDFLAGS) $(LIBS) -DCUOZBLAS -DDOT -DPREC_S testing_cuozblas_scg: $(SRC_SS) $(CXX) $< -o $@ $(FLAGS) $(INCS) $(LDFLAGS) $(LIBS) -DCUOZBLAS -DCG -DPREC_S testing_cuozblas_dgemm: $(SRC_L3) $(CXX) $< -o $@ $(FLAGS) $(INCS) $(LDFLAGS) $(LIBS) -DCUOZBLAS -DGEMM -DPREC_D testing_cuozblas_dgemv: $(SRC_L2) $(CXX) $< -o $@ $(FLAGS) $(INCS) $(LDFLAGS) $(LIBS) -DCUOZBLAS -DGEMV -DPREC_D testing_cuozblas_dcsrmv: $(SRC_L2S) $(CXX) $< -o $@ $(FLAGS) $(INCS) $(LDFLAGS) $(LIBS) -DCUOZBLAS -DCSRMV -DPREC_D testing_cuozblas_ddot: $(SRC_L1B) $(CXX) $< -o $@ $(FLAGS) $(INCS) $(LDFLAGS) $(LIBS) -DCUOZBLAS -DDOT -DPREC_D testing_cuozblas_dcg: $(SRC_SS) $(CXX) $< -o $@ $(FLAGS) $(INCS) $(LDFLAGS) $(LIBS) -DCUOZBLAS -DCG -DPREC_D #testing_cuozblas_sdgemm: $(SRC_L3) # $(CXX) $< -o $@ $(FLAGS) $(INCS) $(LDFLAGS) $(LIBS) -DCUOZBLAS -DGEMM -DPREC_SD #testing_cuozblas_sdgemv: $(SRC_L2) # $(CXX) $< -o $@ $(FLAGS) $(INCS) $(LDFLAGS) $(LIBS) -DCUOZBLAS -DGEMV -DPREC_SD testing_cuozblas_sdcsrmv: $(SRC_L2S) $(CXX) $< -o $@ $(FLAGS) $(INCS) $(LDFLAGS) $(LIBS) -DCUOZBLAS -DCSRMV -DPREC_SD testing_cuozblas_sddot: $(SRC_L1B) $(CXX) $< -o $@ $(FLAGS) $(INCS) $(LDFLAGS) $(LIBS) -DCUOZBLAS -DDOT -DPREC_SD testing_cuozblas_sdcg: $(SRC_SS) $(CXX) $< -o $@ $(FLAGS) $(INCS) $(LDFLAGS) $(LIBS) -DCUOZBLAS -DCG -DPREC_SD ### OZBLAS ################################################################################################### testing_ozblas_sgemm: $(SRC_L3) $(CXX) $< -o $@ $(FLAGS) $(INCS) $(LDFLAGS) $(LIBS) -DOZBLAS -DGEMM -DPREC_S testing_ozblas_sgemv: $(SRC_L2) $(CXX) $< -o $@ $(FLAGS) $(INCS) $(LDFLAGS) $(LIBS) -DOZBLAS -DGEMV -DPREC_S testing_ozblas_scsrmv: $(SRC_L2S) $(CXX) $< -o $@ $(FLAGS) $(INCS) $(LDFLAGS) $(LIBS) -DOZBLAS -DCSRMV -DPREC_S testing_ozblas_sdot: $(SRC_L1B) $(CXX) $< -o $@ $(FLAGS) $(INCS) $(LDFLAGS) $(LIBS) -DOZBLAS -DDOT -DPREC_S testing_ozblas_scg: $(SRC_SS) $(CXX) $< -o $@ $(FLAGS) $(INCS) $(LDFLAGS) $(LIBS) -DOZBLAS -DCG -DPREC_S testing_ozblas_dgemm: $(SRC_L3) $(CXX) $< -o $@ $(FLAGS) $(INCS) $(LDFLAGS) $(LIBS) -DOZBLAS -DGEMM -DPREC_D testing_ozblas_dgemv: $(SRC_L2) $(CXX) $< -o $@ $(FLAGS) $(INCS) $(LDFLAGS) $(LIBS) -DOZBLAS -DGEMV -DPREC_D testing_ozblas_dcsrmv: $(SRC_L2S) $(CXX) $< -o $@ $(FLAGS) $(INCS) $(LDFLAGS) $(LIBS) -DOZBLAS -DCSRMV -DPREC_D testing_ozblas_ddot: $(SRC_L1B) $(CXX) $< -o $@ $(FLAGS) $(INCS) $(LDFLAGS) $(LIBS) -DOZBLAS -DDOT -DPREC_D testing_ozblas_dcg: $(SRC_SS) $(CXX) $< -o $@ $(FLAGS) $(INCS) $(LDFLAGS) $(LIBS) -DOZBLAS -DCG -DPREC_D #testing_ozblas_sdgemm: $(SRC_L3) # $(CXX) $< -o $@ $(FLAGS) $(INCS) $(LDFLAGS) $(LIBS) -DOZBLAS -DGEMM -DPREC_SD #testing_ozblas_sdgemv: $(SRC_L2) # $(CXX) $< -o $@ $(FLAGS) $(INCS) $(LDFLAGS) $(LIBS) -DOZBLAS -DGEMV -DPREC_SD testing_ozblas_sdcsrmv: $(SRC_L2S) $(CXX) $< -o $@ $(FLAGS) $(INCS) $(LDFLAGS) $(LIBS) -DOZBLAS -DCSRMV -DPREC_SD testing_ozblas_sddot: $(SRC_L1B) $(CXX) $< -o $@ $(FLAGS) $(INCS) $(LDFLAGS) $(LIBS) -DOZBLAS -DDOT -DPREC_SD testing_ozblas_sdcg: $(SRC_SS) $(CXX) $< -o $@ $(FLAGS) $(INCS) $(LDFLAGS) $(LIBS) -DOZBLAS -DCG -DPREC_SD clean: $(RM) $(OBJS) $(RM) *.d -include *.d .PHONY: all clean ozblas-1.2a17/testing/testing_common.cpp0000644000175200017520000005012013736547247020677 0ustar mukunokimukunoki// ----- testing handler --------------------------------------------------------------- struct testingHandle_t { // device parameter (single GPU only) #if defined (CUDA) cudaDeviceProp devprop; #endif uint32_t devno; int32_t cc; double device_bytes_per_sec; double device_flops_per_sec_sp; double device_flops_per_sec_dp; int32_t Cores; int32_t DPCores; // testing parameter char mode; // c=check, p=perf double routine_flops; double routine_bytes; int32_t driverVersion; int32_t runtimeVersion; int32_t nodisp; uint32_t numSplitArraysMax; uint32_t fastmode; uint32_t repromode; uint32_t summode; uint32_t useGemmFlag; uint32_t useMyGemmFlag; uint32_t useBatchedGemmFlag; float sparseThreshold; uint32_t trueres; uint32_t verbose; // dim int32_t dim_start; int32_t dim_stop; int32_t dim_step; int32_t dim_m_const; int32_t dim_n_const; int32_t dim_k_const; int32_t dim_m_dev; int32_t dim_n_dev; int32_t dim_k_dev; int32_t dim_m_hst; int32_t dim_n_hst; int32_t dim_k_hst; int32_t incx; int32_t incy; int32_t lda_dev; int32_t ldb_dev; int32_t ldc_dev; uint32_t dim_step_mode; #if defined (CUDA) cublasPointerMode_t pmode; // cublasAtomicsMode_t amode; #endif // options char tranA; // N, T, C char tranB; // N, T, C char mtx_file[256]; int32_t phi; // for input data uint32_t maxiter; // for iterative solvers }; // ----- CUBLAS const value function --------------------------------------------------------------- #if defined (CUDA) inline cublasOperation_t cublas_trans_const_m (char tran) { if (tran == 'N' || tran == 'n') return CUBLAS_OP_N; if (tran == 'T' || tran == 't') return CUBLAS_OP_T; if (tran == 'C' || tran == 'c') return CUBLAS_OP_C; return CUBLAS_OP_N; //default } inline cusparseOperation_t cusparse_trans_const_m (char tran) { if (tran == 'N' || tran == 'n') return CUSPARSE_OPERATION_NON_TRANSPOSE; if (tran == 'T' || tran == 't') return CUSPARSE_OPERATION_TRANSPOSE; if (tran == 'C' || tran == 'c') return CUSPARSE_OPERATION_CONJUGATE_TRANSPOSE; return CUSPARSE_OPERATION_NON_TRANSPOSE; //default } #endif #if !defined (CG) void get_routine_theoretial_performance (testingHandle_t *th) { int32_t s = sizeof(FP_TYPE); int32_t n = th->dim_n_dev; int32_t m = th->dim_m_dev; int32_t k = th->dim_k_dev; th->routine_flops = (double) ROUTINE_FLOPS (m, n, k); th->routine_bytes = (double) ROUTINE_BYTES (m, n, k, s); } #endif void dim_hst_setup (testingHandle_t *th) { th->dim_m_hst = th->dim_stop; th->dim_n_hst = th->dim_stop; th->dim_k_hst = th->dim_stop; if (th->dim_m_const > 0) th->dim_m_hst = th->dim_m_const; if (th->dim_n_const > 0) th->dim_n_hst = th->dim_n_const; if (th->dim_k_const > 0) th->dim_k_hst = th->dim_k_const; } void dim_dev_setup (testingHandle_t *th) { th->dim_m_dev = th->dim_start; th->dim_n_dev = th->dim_start; th->dim_k_dev = th->dim_start; if (th->dim_m_const > 0) th->dim_m_dev = th->dim_m_const; if (th->dim_n_const > 0) th->dim_n_dev = th->dim_n_const; if (th->dim_k_const > 0) th->dim_k_dev = th->dim_k_const; } void dim_dev_increment (testingHandle_t *th) { if (th->dim_m_const == 0) { switch (th->dim_step_mode) { case 1: th->dim_m_dev *= 10; break; case 2: th->dim_m_dev *= 2; break; default: th->dim_m_dev += th->dim_step; break; } } if (th->dim_n_const == 0) { switch (th->dim_step_mode) { case 1: th->dim_n_dev *= 10; break; case 2: th->dim_n_dev *= 2; break; default: th->dim_n_dev += th->dim_step; break; } } if (th->dim_k_const == 0) { switch (th->dim_step_mode) { case 1: th->dim_k_dev *= 10; break; case 2: th->dim_k_dev *= 2; break; default: th->dim_k_dev += th->dim_step; break; } } } __inline__ double gettime () { struct timeval tv; #if defined (CUDA) cudaDeviceSynchronize (); #endif gettimeofday (&tv, NULL); return tv.tv_sec + (double) tv.tv_usec * 1.0e-6; } int32_t get_num_sp_cores (int32_t cc) { switch (cc) { case 300: return 192; case 320: return 192; case 350: return 192; case 370: return 192; case 500: return 128; case 520: return 128; case 530: return 128; case 600: return 64; case 610: return 128; case 620: return 128; case 700: return 64; default: printf ("error: unknown architecture\n"); return 0; } } int32_t get_num_dp_cores (int32_t cc) { switch (cc) { case 300: return 64; case 320: return 64; case 350: return 64; case 370: return 64; case 500: return 0; case 520: return 4; case 530: return 0; case 600: return 32; case 610: return 4; case 620: return 4; case 700: return 32; default: printf ("error: unknown architecture\n"); return 0; } } void dev_setting (testingHandle_t *th) { int32_t driverVersion = 0; int32_t runtimeVersion = 0; int32_t devcnt; #if defined (CUDA) cudaGetDeviceCount (&devcnt); cudaDeviceProp devprop; cudaDriverGetVersion(&driverVersion); cudaRuntimeGetVersion(&runtimeVersion); th->driverVersion = driverVersion; th->runtimeVersion = runtimeVersion; cudaSetDevice (th->devno); cudaGetDeviceProperties (&devprop, th->devno); th->devprop = devprop; th->cc = devprop.major * 100 + devprop.minor * 10; th->Cores = get_num_sp_cores (th->cc); th->DPCores = get_num_dp_cores (th->cc); th->device_bytes_per_sec = (double)th->devprop.memoryClockRate * 2. * 1000. * (double)th->devprop.memoryBusWidth / 8.; th->device_flops_per_sec_sp = (double)th->devprop.clockRate * 1000. * 2. * (double)th->devprop.multiProcessorCount * th->Cores; th->device_flops_per_sec_dp = (double)th->devprop.clockRate * 1000. * 2. * (double)th->devprop.multiProcessorCount * th->DPCores; #endif } void testingCreate (int32_t argc, char **argv, testingHandle_t *th) { // default --------------------------- th->devno = 0; th->mode = 'c'; // options th->tranA = 'N'; th->tranB = 'N'; th->nodisp = 0; th->numSplitArraysMax = 0; th->fastmode = 0; th->repromode = 1; th->summode = 1; th->useGemmFlag = 1; th->useMyGemmFlag = 1; th->useBatchedGemmFlag = 1; th->sparseThreshold = 0.; th->phi = 1; th->maxiter = 100; th->trueres = 0; th->verbose = 10; th->dim_start = 128; th->dim_stop = 1024; th->dim_step = 128; th->dim_m_dev = 0; th->dim_n_dev = 0; th->dim_k_dev = 0; th->dim_m_hst = 0; th->dim_n_hst = 0; th->dim_k_hst = 0; th->dim_m_const = 0; th->dim_n_const = 0; th->dim_k_const = 0; th->incx = 1; th->incy = 1; th->lda_dev = 0; th->ldb_dev = 0; th->ldc_dev = 0; th->dim_step_mode = 0; strcpy (th->mtx_file, ""); #if defined (CUDA) th->pmode = CUBLAS_POINTER_MODE_VALUE; #endif // args --------------------------- int32_t opt, opt_index; optind = 1; struct option long_options[] = { {"transa", required_argument, 0, 'a'}, {"transb", required_argument, 0, 'b'}, {"usebatchedgemm", required_argument, NULL, 'c'}, {"degree", required_argument, 0, 'd'}, {"phi", required_argument, NULL, 'e'}, {"mtx", required_argument, NULL, 'f'}, {"devno", required_argument, NULL, 'g'}, {"repromode", required_argument, NULL, 'h'}, {"fastmode", required_argument, NULL, 'i'}, {"mnk", required_argument, NULL, 'j'}, {"k", required_argument, NULL, 'k'}, {"trueres", required_argument, NULL, 'l'}, {"m", required_argument, NULL, 'm'}, {"n", required_argument, NULL, 'n'}, {"mode", required_argument, NULL, 'o'}, {"maxiter", required_argument, NULL, 'q'}, {"range", required_argument, NULL, 'r'}, {"sparsethreshold", required_argument, NULL, 's'}, {"summode", required_argument, NULL, 't'}, {"verbose", required_argument, NULL, 'u'}, {"nodisp", required_argument, NULL, 'w'}, {"usegemm", required_argument, NULL, 'x'}, {"usemygemm", required_argument, NULL, 'y'}, {0, 0, 0, 0} }; while ((opt = getopt_long (argc, argv, "abcdefghijklmnoqrstuwxy", long_options, &opt_index)) != -1){ switch (opt) { case 'a':{ // N, T, C if (optarg[0]) th->tranA= optarg[0]; break; } case 'b':{ // N, T, C if (optarg[0]) th->tranB= optarg[0]; break; } case 'c':{ th->useBatchedGemmFlag = atoi(optarg); break; } case 'd':{ th->numSplitArraysMax = atoi(optarg); break; } case 'e':{ th->phi = atoi(optarg); break; } case 'f':{ strcpy (th->mtx_file, optarg); break; } case 'g':{ th->devno = atoi(optarg); break; } case 'h':{ th->repromode = atoi(optarg); break; } case 'i':{ th->fastmode = atoi(optarg); break; } case 'j':{ th->dim_start = atoi (optarg); th->dim_stop = atoi (optarg); th->dim_step = 1; break; } case 'k':{ th->dim_k_const = atoi(optarg); break; } case 'l':{ th->trueres = atoi(optarg); break; } case 'm':{ th->dim_m_const = atoi(optarg); break; } case 'n':{ th->dim_n_const = atoi(optarg); break; } case 'o':{ th->mode = optarg[0]; break; } case 'q':{ th->maxiter = atoi(optarg); break; } case 'r':{ char *str, *c_start, *c_stop, *c_step; char par[] = ":"; str = optarg; c_start = strtok (str, par); c_stop = strtok (NULL, par); c_step = strtok (NULL, par); th->dim_start = atoi (c_start); th->dim_stop = atoi (c_stop); if (strcmp (c_step, "pow") == 0) th->dim_step_mode = 2; else th->dim_step = atoi (c_step); break; } case 's':{ th->sparseThreshold = (float)atof(optarg); break; } case 't':{ th->summode = atoi(optarg); break; } case 'u':{ th->verbose = atoi(optarg); break; } case 'w':{ th->nodisp = atoi(optarg); break; } case 'x':{ th->useGemmFlag = atoi(optarg); break; } case 'y':{ th->useMyGemmFlag = atoi(optarg); break; } } } dev_setting (th); } // ========================================= // Print floating-point value with bit representation // ========================================= typedef union{ double d; uint64_t i; } d_and_i; void printBitsB64 (double val) { d_and_i di; di.d = val; // sign printf ("%d", (int)((di.i >> 63) & 1)); printf ("|"); // exponent for (int i = 62; i >= 62-10; i--) printf ("%d", (int)((di.i >> i) & 1)); printf ("|"); // fraction for (int i = 62-11; i >= 0; i--) printf ("%d", (int)((di.i >> i) & 1)); printf (" : "); printf ("%a, %+1.18e\n", val, val); } #if defined (PREC_Q) typedef union{ __float128 q; __int128 i; } q_and_i; void printBitsB128 (__float128 val) { q_and_i qi; qi.q = val; // sign printf ("%d", (int)((qi.i >> 127) & 1)); printf ("|"); // exponent for (int i = 126; i >= 126-14; i--) printf ("%d", (int)((qi.i >> i) & 1)); printf ("|"); // fraction for (int i = 126-15; i >= 0; i--) printf ("%d", (int)((qi.i >> i) & 1)); printf (" : "); char buf[128]; quadmath_snprintf(buf, sizeof(buf), "%.40Qf", val); puts(buf); } #endif void mublasInitMat (int32_t m, int32_t n, int32_t ld, FP_TYPE *mat, double phi, int32_t mode) { int32_t i, j; double erange; FP_TYPE val, valf, r1, r2, x1; FP_TYPE mu = 0.0; FP_TYPE sigma = 1.0; FP_TYPE amax = 0., amin = 1.; if (ld < m) ld = m; switch (mode) { case 1: // init with constant #pragma omp parallel for private (i) for (j = 0; j < n; j++) { for (i = 0; i < m; i++) { mat[j*ld+i] = val = phi; if (amax < fabs(val)) amax = fabs(val); if (amin > fabs(val)) amin = fabs(val); } } erange = log10(amax)-log10(amin); printf ("# input range: %.2f (max = %1.3e, min = %1.3e) (* initialized with const-mode)\n", (isnan(erange)) ? 0 : erange, amax, amin); break; case 2: // phi-mode srand48(123); for (j = 0; j < n; j++) { for (i = 0; i < m; i++) { r1 = drand48(); r2 = drand48(); x1 = mu + (std::sqrt (-2. * log(r1)) * sin (2. * M_PI * r2)) * sigma; val = (r1 - 0.5) * exp (phi * x1); mat[j*ld+i] = (FP_TYPE)val; if (amax < fabs(val)) amax = fabs(val); if (amin > fabs(val)) amin = fabs(val); } } erange = log10(amax)-log10(amin); printf ("# input range: %.2f (max = %1.3e, min = %1.3e) (* initialized with phi-mode)\n", (isnan(erange)) ? 0 : erange, amax, amin); break; case 3: // erange-mode : initialize with +-1e(phi/2) FP_TYPE pi_; #if defined (PREC_Q) pi_ = M_PIq; #elif defined (PREC_DD) pi_ = dd_real::_pi; #elif defined (PREC_MPFR) mpfr_const_pi (pi_, MPFR_RNDN); #else pi_ = M_PI; #endif for (j = 0; j < n; j++) { for (i = 0; i < m; i++) { // double d = drand48()*9.+1.; // d=rand[1:10) // int e = rand() % phi; // [0:phi-1] // val = d * pow(10., e); val = (drand48()*9+1) * pow(10,rand()%(int32_t)phi); #if defined (PREC_DD) valf = dd_real(val,0.); #else valf = val; #endif int32_t s = (rand() % 2) ? 1:-1; valf = s * valf * (pi_/(double)M_PI); mat[j*ld+i] = valf; // min, max #if defined (PREC_DD) val = valf.x[0]; #else val = (double)valf; #endif if (amax < fabs(val)) amax = fabs(val); if (amin > fabs(val)) amin = fabs(val); /* printf ("%d: ",j*n+i); #if defined (PREC_DD) printBitsB64 (valf.x[0]); printBitsB64 (valf.x[1]); #elif defined (PREC_Q) printBitsB128 (valf); #else printBitsB64 (valf); #endif */ } } erange = log10(amax)-log10(amin); printf ("# input range: %.2f (max = %1.3e, min = %1.3e) (* initialized with erange-mode)\n", (isnan(erange)) ? 0 : erange, amax, amin); break; } } void mublasConvMat (int32_t rd, int32_t cd, int32_t ld, FP_TYPE *src, mpreal *dst) { int32_t i, j; if (ld == 0) ld = rd; #pragma omp parallel for private (i) for (j = 0; j < cd; j++) { for (i = 0; i < rd; i++) mpfr_set_d (dst[j*ld+i], src[j*ld+i], MPFR_RNDN); } } void mublasCopyMat (int32_t rd, int32_t cd, int32_t ld, FP_TYPE *src, FP_TYPE *dst) { int32_t i, j; if (ld == 0) ld = rd; #pragma omp parallel for private (i) for (j = 0; j < cd; j++) { for (i = 0; i < rd; i++) dst[j*ld+i] = src[j*ld+i]; } } // check routine (|trg-ref|/|ref|) ------------------------------------------------- void mublasCheckMatrix (int32_t m, int32_t n, FP_TYPE *ftarget, int32_t ldt, mpreal *mpref, int32_t ldr) { FP_TYPE target, ref, dif, rdif; FP_TYPE max = 0., rmax = 0.; size_t errcnt = 0; size_t rerrcnt = 0; for(int32_t j = 0; j < n; j++) { for(int32_t i = 0; i < m; i++) { target = ftarget[j*ldt+i]; ref = mpfr_get_d (mpref[j*ldr+i], MPFR_RNDN); dif = fabs (target - ref); if (isinf(dif) || isnan(dif)) { printf("\terr: target or reference includes inf or nan\n"); exit(1); } rdif = (dif == 0 || ref == 0) ? 0. : dif / ref; // printf(" -- [%d,%d] trg=%1.3e ref=%1.3e diff=%1.3e (%1.3e)\n", j, i, target, ref, dif, rdif); if (max < dif) max = dif; if (rmax < rdif) rmax = rdif; if (dif != 0) errcnt++; if (rdif != 0) rerrcnt++; } } printf("\t%1.6e\t%zd\t%1.6e\t%zd\t", max, errcnt, rmax, rerrcnt); printf("## max(|t-r|)\terrcnt\tmax(|t-r|/r)\terrcnt"); } // print routine ------------------------------------------------- void print_routine_name () { #if defined (PREC_S) printf ("S"); #elif defined (PREC_SD) printf ("SD"); #elif defined (PREC_D) printf ("D"); #else printf ("unknown-precision,"); #endif #if defined (DOT) printf ("DOT\n"); #elif defined (GEMV) printf ("GEMV\n"); #elif defined (CSRMV) printf ("CSRMV\n"); #elif defined (GEMM) printf ("GEMM\n"); #elif defined (CG) printf ("CG\n"); #else printf ("unknown-routine\n"); #endif } void print_library_name () { #if defined (CUOZBLAS) printf ("CUOZBLAS (using cuBLAS)\n"); #elif defined (OZBLAS) printf ("OZBLAS"); #if defined (MKL) printf (" (using MKL"); #elif defined (OpenBLAS) printf (" (using OpenBLAS"); #endif #if defined (NVBLAS) printf (" with NVBLAS)\n"); #else printf (")\n"); #endif #else printf ("unknown-library\n"); #endif } #include void print_info1 (testingHandle_t *th) { if (!th->nodisp) { system("echo -n '# Testing info -----------------------------\n'"); system("echo -n '# \tDate:\t'; date"); system("echo -n '# \tCPU:'; cat /proc/cpuinfo | grep 'model name' | head -n 1"); system("echo '# \t - OMP_NUM_THREADS = ' $OMP_NUM_THREADS"); system("echo '# \t - MKL_NUM_THREADS = ' $MKL_NUM_THREADS"); system("echo -n '# \tOS:\t'; uname -a"); system("echo -n '# \tHost:\t'; hostname"); #if defined (CUDA) && defined (CUOZBLAS) || defined (NVBLAS) printf("# \tDevice[%d]\t%s\n", th->devno, th->devprop.name); printf("# \t- Theoretical Peak Memory Bandwidth:\t%3.1f [GB/s]\n", th->device_bytes_per_sec * 1.e-9); printf("# \t- Theoretical Peak Performance (SP):\t%4.1f [GFlops]\n", th->device_flops_per_sec_sp * 1.e-9); printf("# \t- Theoretical Peak Performance (DP):\t%4.1f [GFlops]\n", th->device_flops_per_sec_dp * 1.e-9); printf("# \t- Total Gmem:\t%4.1f [GB]\n", th->devprop.totalGlobalMem * 1e-9); printf("# \t- CC:\t%d.%d\n", th->devprop.major, th->devprop.minor); printf("# \t- ECC Enabled:\t%d\n", th->devprop.ECCEnabled); printf("# \t- MultiProcessorCount:\t%d\n", th->devprop.multiProcessorCount); printf("# \tDriver / Runtime:\t%d, %d\n", th->driverVersion, th->runtimeVersion); #endif printf ("#\tRoutine:\t"); print_routine_name (); printf ("#\tLibrary:\t"); print_library_name (); #if !defined (CG) if (th->mode == 'c') { printf ("#\tmode: checking with "); printf ("MBLASMPFR with %zd-bit\n", mpfr_get_default_prec()); } else { printf ("#\tmode: performance evaluation\n"); } printf ("#\tNLOOP/WLOOP:\t%d/%d\n", NLOOP, WLOOP); #endif printf ("# ------------------------------------------\n"); #if defined (CUOZBLAS) || defined (OZBLAS) printf ("# OzBLAS -----------------------------------\n"); printf ("#\tWork-mem size = %1.1e (bytes)\n", WORK_MEM_SIZE); printf ("#\tDegree = %d\n", th->numSplitArraysMax); printf ("#\tFastMode = %d\n", th->fastmode); printf ("#\tReproMode = %d\n", th->repromode); printf ("#\tSumMode = %d\n", th->summode); printf ("#\tUseGemmFlag = %d\n", th->useGemmFlag); printf ("#\tUseMyGemmFlag = %d\n", th->useMyGemmFlag); printf ("#\tUseBatchedGemmFlag = %d\n", th->useBatchedGemmFlag); printf ("#\tSparseThreshold (GEMM) = %f\n", th->sparseThreshold); printf ("# ------------------------------------------\n"); #endif printf ("# BLAS parameter ---------------------------\n"); printf ("#\ttranA:\t%c\n", th->tranA); printf ("#\ttranB:\t%c\n", th->tranB); printf ("# Problem setting --------------------------\n"); printf ("#\tExpRange = [1e0:1e%d)\n", th->phi); #if !defined (CSRMV) printf ("#\tdim_start:\t%d\n", th->dim_start); printf ("#\tdim_stop:\t%d\n", th->dim_stop); printf ("#\tdim_step:\t%d\n", th->dim_step); if (th->dim_m_const == 0) printf ("#\tM:\trange\n"); else printf ("#\tM:\t%d\n", th->dim_m_const); if (th->dim_n_const == 0) printf ("#\tN:\trange\n"); else printf ("#\tN:\t%d\n", th->dim_n_const); if (th->dim_k_const == 0) printf ("#\tK:\trange\n"); else printf ("#\tK:\t%d\n", th->dim_k_const); #endif #if defined (CUDA) if (th->pmode) printf ("#\tcublaspointer:\tDEVICE\n"); else printf ("#\tcublaspointer:\tHOST\n"); #endif printf ("# ------------------------------------------\n"); } } void print_info2 (testingHandle_t *th) { if (!th->nodisp) { printf ("# Evaluation result ------------------------\n"); if (th->mode == 'p') printf ("(* shows the value at the last execution, ** shows the value of the last block at the last execution)\n"); printf ("# M\tN\tK"); if (th->mode == 'p') printf ("\tTime[sec]\tPerf[GFlops]\tPerf[GB/s]\titr\t*t_SpltA\t*t_SpltB\t*t_Comp \t*t_Sum \t*t_Other\t*t_Total\t**#SpltA\t**#SpltB\t**Mem[MB]"); else printf ("\tERROR\t#ofERROR"); printf ("\n"); } } #if defined (CUOZBLAS) void print_cuozblas_info1 (testingHandle_t *th, cuozblasHandle_t *ha) { if (th->mode == 'p') { double t_other = ha->t_total - ha->t_SplitA - ha->t_SplitB - ha->t_comp - ha->t_sum; printf ("\t%1.3e\t%1.3e\t%1.3e\t%1.3e\t%1.3e\t%1.3e", ha->t_SplitA, ha->t_SplitB, ha->t_comp, ha->t_sum, t_other, ha->t_total); printf ("\t%d\t%d\t%1.3e", ha->numSplitArraysA, ha->numSplitArraysB, ha->memAddr*1.e-6); } } #elif defined (OZBLAS) void print_ozblas_info1 (testingHandle_t *th, ozblasHandle_t *ha) { if (th->mode == 'p') { double t_other = ha->t_total - ha->t_SplitA - ha->t_SplitB - ha->t_comp - ha->t_sum; printf ("\t%1.3e\t%1.3e\t%1.3e\t%1.3e\t%1.3e\t%1.3e", ha->t_SplitA, ha->t_SplitB, ha->t_comp, ha->t_sum, t_other, ha->t_total); printf ("\t%d\t%d\t%1.3e", ha->numSplitArraysA, ha->numSplitArraysB, ha->memAddr*1.e-6); } } #endif ozblas-1.2a17/LICENSE0000644000175200017520000000220213725177602014464 0ustar mukunokimukunokiThe MIT License (MIT) Copyright (c) 2018-2019 Daichi Mukunoki (Tokyo Woman's Christian University) Copyright (c) 2019-2020 RIKEN Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ozblas-1.2a17/make.inc0000644000175200017520000000310213736547327015076 0ustar mukunokimukunoki############################################## #--- System ---------------------------------- #SYSTEM = OFP #CPU_ARCH = KNL CPU_ARCH = SKYLAKE #--- PATH ------------------------------------ # To build (cu)ozblas.a CUDA_PATH = /usr/local/cuda-10.2 CBLAS_INC = $(HOME)/local/include # -> the location of "cblas.h" # To build testing programs MPACK_DIR = $(HOME)/local MPACK_LDFLAGS = -L$(MPACK_DIR)/lib MPACK_INC = -I$(MPACK_DIR)/include/mpack MPFR_DIR = $(HOME)/local MPFR_LDFLAGS = -L$(MPFR_DIR)/lib MPFR_INC = -I$(MPFR_DIR)/include #--- Compiler -------------------------------- CXX = icpc # -> icpc or g++ # Note: do not mix intel/gcc OpenMP #--- CPU (OzBLAS) / CUDA (cuOzBLAS) ---------- CPU = yes CUDA = yes #--- BLAS (CPU) ------------------------------ CPU_BLAS = MKL # -> MKL or OpenBLAS # CSRMV and CG require MKL as OpenBLAS does not have sparse routines. NVBLAS = no # -> yes or no. if yes, it uses NVBLAS routines instead of MKL or OpenBLAS on CPUs #--- CUDA Compute Capability ----------------- CC30 = no CC32 = no CC35 = no CC37 = no CC50 = no CC52 = no CC53 = no CC60 = no CC61 = no CC62 = no CC70 = yes CC75 = no #--- Precision ------------------------------- PREC_D = yes PREC_S = yes PREC_SD = yes #--- Routine --------------------------------- # level-1 DOT = yes NRM2 = yes AXPY = yes # level-2 GEMV = yes CSRMV = yes # -> CSRMV requires MKL on CPUs # -> CSRMV requires MAGMA for testing # level-3 GEMM = yes # solver CG = yes # -> CG requires CSRMV, DOT, NRM2, and AXPY ############################################## ozblas-1.2a17/README0000644000175200017520000001177013736552046014352 0ustar mukunokimukunokiOzBLAS 1.2 alpha v.17 October 5, 2020 Daichi Mukunoki, Takeshi Ogita, Katsuhisa Ozaki Contact: Daichi Mukunoki (daichi.mukunoki@riken.jp) @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ ATTENTION! Basically, this code is just intended to demonstrate our proposed methods presented in our papers. It is not recommended to use it in real applications. The behavior of our implementation may depend on the compiler (options) and the version of the libraries used internally. If you want to use it in real applications, please contact us. @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ Note: - This version is tested only using Intel MKL and Intel compiler. Other BLAS and compilers are not tested (in theory, the others can be used, but it may require to set some compiler options). Note (for developers): - For CG solvers: on CPU, (1) usegemm=0, (2) splitShift does not change NumSplitMat but try to increase the bit-amount on SplitMat (to reduce NumSplitVec). On GPU, (2) usegemm=1, (3) splitShift decreases NumSplitMat. (1) Overview Accurate and reproducible BLAS routines "OzBLAS" [3] based on Ozaki scheme [1] and NearSum [2]. This version contains following routines: DOT, NRM2, GEMV, GEMM, AXPY, CSRMV, and CG solver (unpreconditioned) on single and double-precision for NVIDIA GPUs and CPUs. The Ozaki scheme is applied only to the inner product operations in the BLAS operation. The computation associated with scalar parameters (alpha and beta) are performed on the standard double-precision operation. The computation associated with scalar parameters and DAXPY are performed with fused-multiply-add (FMA) (the use is forced). The CPU version uses Intel MKL for the internal computation (probably, other BLAS such as OpenBLAS and NVBLAS can be used, but not tested). The GPU version uses NVIDIA cuBLAS for the internal computation. This code is an experimental alpha version. We do not guarantee the computation result. However, we welcome your feedback. (2) Requirements: libozblas.a: - Intel compiler and MKL libcuozblas.a: - cuBLAS and CUDA testing: - The GNU MPFR Library (https://www.mpfr.org) - The MPACK; Multiple precision arithmetic BLAS (MBLAS) and LAPACK (MLAPACK) (http://mplapack.sourceforge.net) (3) Build: After modifying Makefiles in src and testing directories, execute make in the current directory. (4) Usage: Compile your code with the following files: CPU: include/ozblas.h lib/libozblas.a GPU: include/cuozblas.h lib/libcuozblas.a In your code, (cu)ozblasHandler must be declared and used. You must call (cu)ozblasCreate and (cu)ozblasDestroy before and after the use of the BLAS routines, respectively. For example, ozblasHandle_t ha; ozblasCreate (&ha, workMemSize, numSplitArraysMax, fastMode, reproMode, sumMode); ozblasDgemv (&ha, trans, m, n, alpha, a, lda, x, 1, beta, y, 1); ozblasDestroy (&ha); - workMemSize (size_t): work memory size in bytes. - numSplitArraysMax (uint): computation degree (i.e., the maximum number of split matrices/vectors used in the Ozaki scheme). If 0 (default), the maximum number required to achieve the maximum accuracy is automatically used. - fastMode (uint): use/non-use of fastmode (1: use, 0: non-use). Fastmode may improve the performance on compute-bound operations (such as GEMM) with a degradation of accuracy. - reproMode (uint): ensuring the reproducibility of the computation result (1: ensured, 0: not ensured). - sumMode (uint): the computation mode of the summation (1: NearSum, 0: standard floating-point arithmetic). Also, please check and modify "testing/testing_setting.h" before evaluation. (5) Limitations: - CSRMV supports 32-bit indexing only (so, MAGMA must be build with 32-bit indexing). - incx and incy are not supported. (6) References: [1] Ozaki K., Ogita T., Oishi S., Rump S. M.: Error-free transformations of matrix multiplication by using fast routines of matrix multiplication and its applications, Numer. Algorithms, vol. 59, no. 1, pp. 95-118, 2012. [2] Rump S., Ogita T., Oishi S.: Accurate Floating-Point Summation Part II: Sign, K-Fold Faithful and Rounding to Nearest, SIAM Journal on Scientific Computing, Vol. 31, No. 2, pp. 1269-1302, 2008. [3] D. Mukunoki, T. Ogita, K. Ozaki: Accurate and Reproducible BLAS Routines with Ozaki Scheme for Many-core Architectures, Proc. 13th International Conference on Parallel Processing and Applied Mathematics (PPAM2019), LNCS, Vol. 12043, pp. 516-527, 2019. (7) History: v1.2a17 Performance improvement of the CPU implementation. Bug fix (previously, reproducibility was not ensured in some cases between CPU and GPU). Some minor updates. v1.2a Unpreconditioned CG solver is added. CSRMV for CPUs is added. Single-precision routines are added. Single-precision routines performed with double are added (partially). A critical bug in the vector splitting on the CPU implementation is fixed. Some minor updates. v1.1a04 Scalar arguments (alpha and beta) are supported. CUBLAS_POINTER_MODE_DEVICE is supported. DNRM2 and DCSRMV for GPUs are added. OpenBLAS and NVBLAS are supported. Some bugs are fixed. v1.0a03 Initial release.