Date: 2022-03-04T17:40:01+00:00

Actual source code: petscdevice.h

      1: #if !defined(PETSCDEVICE_H)
      2: #define PETSCDEVICE_H

      4: #include <petscsys.h>
      5: #include <petscdevicetypes.h>

      7: #if PetscDefined(HAVE_CUDA)
      8: #include <cuda.h>
      9: #include <cuda_runtime.h>
     10: #include <cublas_v2.h>
     11: #include <cusolverDn.h>
     12: #include <cusolverSp.h>
     13: #include <cufft.h>

     15: PETSC_EXTERN cudaEvent_t petsc_gputimer_begin;
     16: PETSC_EXTERN cudaEvent_t petsc_gputimer_end;

     18: /* cuBLAS does not have cublasGetErrorName(). We create one on our own. */
     19: PETSC_EXTERN const char* PetscCUBLASGetErrorName(cublasStatus_t); /* PETSC_EXTERN since it is exposed by the CHKERRCUBLAS macro */
     20: PETSC_EXTERN const char* PetscCUSolverGetErrorName(cusolverStatus_t);
     21: PETSC_EXTERN const char* PetscCUFFTGetErrorName(cufftResult);

     23: #define WaitForCUDA() PetscCUDASynchronize ? cudaDeviceSynchronize() : cudaSuccess;

     25: /* CUDART_VERSION = 1000 x major + 10 x minor version */

     27: /* Could not find exactly which CUDART_VERSION introduced cudaGetErrorName. At least it was in CUDA 8.0 (Sep. 2016) */
     28: #if (CUDART_VERSION >= 8000) /* CUDA 8.0 */
     29: #define CHKERRCUDA(cerr)                                                \
     30:   do {                                                                  \
     31:     if (PetscUnlikely(cerr)) {                                          \
     32:       const char *name  = cudaGetErrorName(cerr);                       \
     33:       const char *descr = cudaGetErrorString(cerr);                     \
     34:       SETERRQ3(PETSC_COMM_SELF,PETSC_ERR_GPU,"cuda error %d (%s) : %s", \
     35:                (int)cerr,name,descr);                                   \
     36:     }                                                                   \
     37:   } while (0)
     38: #else
     39: #define CHKERRCUDA(cerr) do {if (PetscUnlikely(cerr)) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_GPU,"cuda error %d",(int)cerr);} while (0)
     40: #endif /* CUDART_VERSION >= 8000 */

     42: #define CHKERRCUBLAS(stat)                                              \
     43:   do {                                                                  \
     44:     if (PetscUnlikely(stat)) {                                          \
     45:       const char *name = PetscCUBLASGetErrorName(stat);                 \
     46:       if (((stat == CUBLAS_STATUS_NOT_INITIALIZED) || (stat == CUBLAS_STATUS_ALLOC_FAILED)) && PetscCUDAInitialized) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_GPU_RESOURCE,"cuBLAS error %d (%s). Reports not initialized or alloc failed; this indicates the GPU has run out resources",(int)stat,name); \
     47:       else SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_GPU,"cuBLAS error %d (%s)",(int)stat,name); \
     48:     }                                                                   \
     49:   } while (0)

     51: #define CHKERRCUSOLVER(stat)                                            \
     52:   do {                                                                  \
     53:     if (PetscUnlikely(stat)) {                                          \
     54:       const char *name = PetscCUSolverGetErrorName(stat);               \
     55:       if ((stat == CUSOLVER_STATUS_NOT_INITIALIZED) || (stat == CUSOLVER_STATUS_ALLOC_FAILED) || (stat == CUSOLVER_STATUS_INTERNAL_ERROR)) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_GPU_RESOURCE,"cuSolver error %d (%s). This indicates the GPU has run out resources",(int)stat,name); \
     56:       else SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_GPU,"cuSolver error %d (%s)",(int)stat,name); \
     57:     }                                                                   \
     58:   } while (0)

     60: #define CHKERRCUFFT(res)                                                \
     61:   do {                                                                  \
     62:     if (PetscUnlikely(res)) {                                           \
     63:       const char *name = PetscCUFFTGetErrorName(res);                   \
     64:       if (((res == CUFFT_SETUP_FAILED) || (res == CUFFT_ALLOC_FAILED)) && PetscCUDAInitialized) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_GPU_RESOURCE,"cuFFT error %d (%s). Reports not initialized or alloc failed; this indicates the GPU has run out resources",(int)res,name); \
     65:       else SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_GPU,"cuFFT error %d (%s)",(int)res,name); \
     66:     }                                                                   \
     67:   } while (0)

     69: PETSC_EXTERN cudaStream_t   PetscDefaultCudaStream; /* The default stream used by PETSc */
     70: PETSC_INTERN PetscErrorCode PetscCUBLASInitializeHandle(void);
     71: PETSC_INTERN PetscErrorCode PetscCUSOLVERDnInitializeHandle(void);

     73: PETSC_EXTERN PetscErrorCode PetscCUBLASGetHandle(cublasHandle_t*);
     74: PETSC_EXTERN PetscErrorCode PetscCUSOLVERDnGetHandle(cusolverDnHandle_t*);
     75: #endif /* PetscDefined(HAVE_CUDA) */

     77: #if PetscDefined(HAVE_HIP)
     78: #include <hip/hip_runtime.h>
     79: #include <hipblas.h>
     80: #if defined(__HIP_PLATFORM_NVCC__)
     81: #include <cusolverDn.h>
     82: #else /* __HIP_PLATFORM_HCC__ */
     83: #include <rocsolver.h>
     84: #endif /* __HIP_PLATFORM_NVCC__ */

     86: #define WaitForHIP() PetscHIPSynchronize ? hipDeviceSynchronize() : hipSuccess;

     88: PETSC_EXTERN hipEvent_t petsc_gputimer_begin;
     89: PETSC_EXTERN hipEvent_t petsc_gputimer_end;

     91: /* hipBLAS does not have hipblasGetErrorName(). We create one on our own. */
     92: PETSC_EXTERN const char* PetscHIPBLASGetErrorName(hipblasStatus_t); /* PETSC_EXTERN since it is exposed by the CHKERRHIPBLAS macro */

     94: #define CHKERRHIP(cerr)                                                 \
     95:   do {                                                                  \
     96:     if (PetscUnlikely(cerr)) {                                          \
     97:       const char *name  = hipGetErrorName(cerr);                        \
     98:       const char *descr = hipGetErrorString(cerr);                      \
     99:       SETERRQ3(PETSC_COMM_SELF,PETSC_ERR_LIB,"hip error %d (%s) : %s",  \
    100:                (int)cerr,name,descr);                                   \
    101:     }                                                                   \
    102:   } while (0)

    104: #define CHKERRHIPBLAS(stat)                                             \
    105:   do {                                                                  \
    106:     if (PetscUnlikely(stat)) {                                          \
    107:       const char *name = PetscHIPBLASGetErrorName(stat);                \
    108:       SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_LIB,"hipBLAS error %d (%s)",   \
    109:                (int)stat,name);                                         \
    110:     }                                                                   \
    111:   } while (0)

    113: /* TODO: SEK:  Need to figure out the hipsolver issues */
    114: #define CHKERRHIPSOLVER(err)                                            \
    115:   do {                                                                  \
    116:     if (PetscUnlikely(err)) {                                           \
    117:       SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_LIB,"HIPSOLVER error %d",err); \
    118:     }                                                                   \
    119:   } while (0)

    121: /* hipSolver does not exist yet so we work around it
    122:    rocSOLVER users rocBLAS for the handle
    123:  * */
    124: #if defined(__HIP_PLATFORM_NVCC__)
    125: typedef cusolverDnHandle_t hipsolverHandle_t;
    126: typedef cusolverStatus_t   hipsolverStatus_t;

    128: /* Alias hipsolverDestroy to cusolverDnDestroy */
    129: PETSC_STATIC_INLINE hipsolverStatus_t hipsolverDestroy(hipsolverHandle_t *hipsolverhandle)
    130: {
    131:   return cusolverDnDestroy(hipsolverhandle)
    132: }

    134: /* Alias hipsolverCreate to cusolverDnCreate */
    135: PETSC_STATIC_INLINE hipsolverStatus_t hipsolverCreate(hipsolverHandle_t *hipsolverhandle)
    136: {
    137:   return cusolverDnCreate(hipsolverhandle)
    138: }

    140: /* Alias hipsolverGetStream to cusolverDnGetStream */
    141: PETSC_STATIC_INLINE hipsolverStatus_t hipsolverGetStream(hipsolverHandle_t handle, hipStream_t *stream)
    142: {
    143:   return cusolverDnGetStream(handle,stream);
    144: }

    146: /* Alias hipsolverSetStream to cusolverDnSetStream */
    147: PETSC_STATIC_INLINE hipsolverStatus_t hipsolverSetStream(hipsolverHandle_t handle, hipStream_t stream)
    148: {
    149:   return cusolveDnSetStream(handle,stream);
    150: }
    151: #else /* __HIP_PLATFORM_HCC__ */
    152: typedef rocblas_handle hipsolverHandle_t;
    153: typedef rocblas_status hipsolverStatus_t;

    155: /* Alias hipsolverDestroy to rocblas_destroy_handle */
    156: PETSC_STATIC_INLINE hipsolverStatus_t hipsolverDestroy(hipsolverHandle_t  hipsolverhandle)
    157: {
    158:   return rocblas_destroy_handle(hipsolverhandle);
    159: }

    161: /* Alias hipsolverCreate to rocblas_destroy_handle */
    162: PETSC_STATIC_INLINE hipsolverStatus_t hipsolverCreate(hipsolverHandle_t *hipsolverhandle)
    163: {
    164:   return rocblas_create_handle(hipsolverhandle);
    165: }

    167: /* Alias hipsolverGetStream to rocblas_get_stream */
    168: PETSC_STATIC_INLINE hipsolverStatus_t hipsolverGetStream(hipsolverHandle_t handle, hipStream_t *stream)
    169: {
    170:   return rocblas_get_stream(handle,stream);
    171: }

    173: /* Alias hipsolverSetStream to rocblas_set_stream */
    174: PETSC_STATIC_INLINE hipsolverStatus_t hipsolverSetStream(hipsolverHandle_t handle, hipStream_t stream)
    175: {
    176:   return rocblas_set_stream(handle,stream);
    177: }
    178: #endif /* __HIP_PLATFORM_NVCC__ */
    179: PETSC_EXTERN hipStream_t    PetscDefaultHipStream; /* The default stream used by PETSc */
    180: PETSC_INTERN PetscErrorCode PetscHIPBLASInitializeHandle(void);
    181: PETSC_INTERN PetscErrorCode PetscHIPSOLVERInitializeHandle(void);

    183: PETSC_EXTERN PetscErrorCode PetscHIPBLASGetHandle(hipblasHandle_t*);
    184: PETSC_EXTERN PetscErrorCode PetscHIPSOLVERGetHandle(hipsolverHandle_t*);
    185: #endif /* PetscDefined(HAVE_HIP) */

    187: /* Cannot use the device context api without C++11 */
    188: #if PetscDefined(HAVE_CXX_DIALECT_CXX11)
    189: PETSC_EXTERN PetscErrorCode PetscDeviceInitializePackage(void);
    190: PETSC_EXTERN PetscErrorCode PetscDeviceFinalizePackage(void);

    192: /* PetscDevice */
    193: PETSC_EXTERN PetscErrorCode PetscDeviceCreate(PetscDeviceKind,PetscDevice*);
    194: PETSC_EXTERN PetscErrorCode PetscDeviceConfigure(PetscDevice);
    195: PETSC_EXTERN PetscErrorCode PetscDeviceDestroy(PetscDevice*);

    197: /* PetscDeviceContext */
    198: PETSC_EXTERN PetscErrorCode PetscDeviceContextCreate(PetscDeviceContext*);
    199: PETSC_EXTERN PetscErrorCode PetscDeviceContextDestroy(PetscDeviceContext*);
    200: PETSC_EXTERN PetscErrorCode PetscDeviceContextSetDevice(PetscDeviceContext,PetscDevice);
    201: PETSC_EXTERN PetscErrorCode PetscDeviceContextGetDevice(PetscDeviceContext,PetscDevice*);
    202: PETSC_EXTERN PetscErrorCode PetscDeviceContextSetStreamType(PetscDeviceContext,PetscStreamType);
    203: PETSC_EXTERN PetscErrorCode PetscDeviceContextGetStreamType(PetscDeviceContext,PetscStreamType*);
    204: PETSC_EXTERN PetscErrorCode PetscDeviceContextSetUp(PetscDeviceContext);
    205: PETSC_EXTERN PetscErrorCode PetscDeviceContextDuplicate(PetscDeviceContext,PetscDeviceContext*);
    206: PETSC_EXTERN PetscErrorCode PetscDeviceContextQueryIdle(PetscDeviceContext,PetscBool*);
    207: PETSC_EXTERN PetscErrorCode PetscDeviceContextWaitForContext(PetscDeviceContext,PetscDeviceContext);
    208: PETSC_EXTERN PetscErrorCode PetscDeviceContextFork(PetscDeviceContext,PetscInt,PetscDeviceContext**);
    209: PETSC_EXTERN PetscErrorCode PetscDeviceContextJoin(PetscDeviceContext,PetscInt,PetscDeviceContextJoinMode,PetscDeviceContext**);
    210: PETSC_EXTERN PetscErrorCode PetscDeviceContextSynchronize(PetscDeviceContext);
    211: PETSC_EXTERN PetscErrorCode PetscDeviceContextGetCurrentContext(PetscDeviceContext*);
    212: PETSC_EXTERN PetscErrorCode PetscDeviceContextSetCurrentContext(PetscDeviceContext);
    213: PETSC_EXTERN PetscErrorCode PetscDeviceContextSetFromOptions(MPI_Comm,const char[],PetscDeviceContext);
    214: #endif /* PetscDefined(HAVE_CXX_DIALECT_CXX11) */
    215: #endif /* PETSCDEVICE_H */