Date: 2022-03-31T04:49:10+00:00

Actual source code: petscdevicetypes.h

      1: #ifndef PETSCDEVICETYPES_H
      2: #define PETSCDEVICETYPES_H

      4: /* for PETSC_HAVE_CUDA/HIP/KOKKOS etc */
      5: #include <petscsys.h>

      7: /*E
      8:   PetscMemType - Memory type of a pointer

     10:   Developer Note:
     11:   Encoding of the bitmask in binary: xxxxyyyz

     13: $ z = 0                - Host memory
     14: $ z = 1                - Device memory
     15: $ yyy = 000            - CUDA-related memory
     16: $ yyy = 001            - HIP-related memory
     17: $ yyy = 010            - SYCL-related memory
     18: $ xxxxyyy1 = 0000,0001 - CUDA memory
     19: $ xxxxyyy1 = 0001,0001 - CUDA NVSHMEM memory
     20: $ xxxxyyy1 = 0000,0011 - HIP memory
     21: $ xxxxyyy1 = 0000,0101 - SYCL memory

     23:   Other types of memory, e.g., CUDA managed memory, can be added when needed.

     25:   Level: beginner

     27:   Notes:
     28:   PETSC_MEMTYPE_KOKKOS depends on the KOKKOS backend configuration

     30: .seealso: VecGetArrayAndMemType(), PetscSFBcastWithMemTypeBegin(), PetscSFReduceWithMemTypeBegin()
     31: E*/
     32: typedef enum {
     33:   PETSC_MEMTYPE_HOST    = 0,
     34:   PETSC_MEMTYPE_DEVICE  = 0x01,
     35:   PETSC_MEMTYPE_CUDA    = 0x01,
     36:   PETSC_MEMTYPE_NVSHMEM = 0x11,
     37:   PETSC_MEMTYPE_HIP     = 0x03,
     38:   PETSC_MEMTYPE_SYCL    = 0x05,
     39: } PetscMemType;
     40: #if PetscDefined(HAVE_CUDA)
     41: # define PETSC_MEMTYPE_KOKKOS PETSC_MEMTYPE_CUDA
     42: #elif PetscDefined(HAVE_HIP)
     43: # define PETSC_MEMTYPE_KOKKOS PETSC_MEMTYPE_HIP
     44: #elif PetscDefined(HAVE_SYCL)
     45: # define PETSC_MEMTYPE_KOKKOS PETSC_MEMTYPE_SYCL
     46: #else
     47: # define PETSC_MEMTYPE_KOKKOS PETSC_MEMTYPE_HOST
     48: #endif

     50: #define PetscMemTypeHost(m)    (((m) & 0x1) == PETSC_MEMTYPE_HOST)
     51: #define PetscMemTypeDevice(m)  (((m) & 0x1) == PETSC_MEMTYPE_DEVICE)
     52: #define PetscMemTypeCUDA(m)    (((m) & 0xF) == PETSC_MEMTYPE_CUDA)
     53: #define PetscMemTypeHIP(m)     (((m) & 0xF) == PETSC_MEMTYPE_HIP)
     54: #define PetscMemTypeSYCL(m)    (((m) & 0xF) == PETSC_MEMTYPE_SYCL)
     55: #define PetscMemTypeNVSHMEM(m) ((m) == PETSC_MEMTYPE_NVSHMEM)

     57: #define PETSC_OFFLOAD_VECKOKKOS_DEPRECATED PETSC_OFFLOAD_VECKOKKOS PETSC_DEPRECATED_ENUM("Use PETSC_OFFLOAD_KOKKOS (since version 3.17.0)")
     58: /*E
     59:   PetscOffloadMask - indicates which memory (CPU, GPU, or none) contains valid data

     61: $ PETSC_OFFLOAD_UNALLOCATED - no memory contains valid matrix entries; NEVER used for vectors
     62: $ PETSC_OFFLOAD_GPU         - GPU has valid vector/matrix entries
     63: $ PETSC_OFFLOAD_CPU         - CPU has valid vector/matrix entries
     64: $ PETSC_OFFLOAD_BOTH        - Both GPU and CPU have valid vector/matrix entries and they match
     65: $ PETSC_OFFLOAD_KOKKOS      - Reserved for Kokkos matrix and vector. It means the offload is managed by Kokkos, thus this flag itself cannot tell you where the valid data is.

     67:   Level: developer
     68: E*/
     69: typedef enum {
     70:   PETSC_OFFLOAD_UNALLOCATED = 0x0,
     71:   PETSC_OFFLOAD_CPU         = 0x1,
     72:   PETSC_OFFLOAD_GPU         = 0x2,
     73:   PETSC_OFFLOAD_BOTH        = 0x3,
     74:   PETSC_OFFLOAD_VECKOKKOS_DEPRECATED = 0x100,
     75:   PETSC_OFFLOAD_KOKKOS      = 0x100
     76: } PetscOffloadMask;

     78: /*E
     79:   PetscDeviceInitType - Initialization strategy for PetscDevice

     81: $ PETSC_DEVICE_INIT_NONE  - PetscDevice is never initialized
     82: $ PETSC_DEVICE_INIT_LAZY  - PetscDevice is initialized on demand
     83: $ PETSC_DEVICE_INIT_EAGER - PetscDevice is initialized as soon as possible

     85:   Notes:
     86:   PETSC_DEVICE_INIT_NONE implies that any initialization of PetscDevice is disallowed and
     87:   doing so results in an error. Useful to ensure that no accelerator is used in a program.

     89:   Level: beginner

     91: .seealso: PetscDevice, PetscDeviceType, PetscDeviceInitialize(), PetscDeviceInitialized(), PetscDeviceCreate()
     92: E*/
     93: typedef enum {
     94:   PETSC_DEVICE_INIT_NONE,
     95:   PETSC_DEVICE_INIT_LAZY,
     96:   PETSC_DEVICE_INIT_EAGER
     97: } PetscDeviceInitType;
     98: PETSC_EXTERN const char *const PetscDeviceInitTypes[];

    100: /*E
    101:   PetscDeviceType - Kind of accelerator device backend

    103: $ PETSC_DEVICE_INVALID - Invalid type, do not use
    104: $ PETSC_DEVICE_CUDA    - CUDA enabled GPU
    105: $ PETSC_DEVICE_HIP     - ROCM/HIP enabled GPU
    106: $ PETSC_DEVICE_SYCL    - SYCL enabled device
    107: $ PETSC_DEVICE_DEFAULT - Automatically select backend based on availability
    108: $ PETSC_DEVICE_MAX     - Always 1 greater than the largest valid PetscDeviceType, invalid type, do not use

    110:   Notes:
    111:   PETSC_DEVICE_DEFAULT is selected in the following order: PETSC_DEVICE_HIP, PETSC_DEVICE_CUDA, PETSC_DEVICE_SYCL, PETSC_DEVICE_INVALID.

    113:   Level: beginner

    115: .seealso: PetscDevice, PetscDeviceInitType, PetscDeviceCreate()
    116: E*/
    117: typedef enum {
    118:   PETSC_DEVICE_INVALID,
    119:   PETSC_DEVICE_CUDA,
    120:   PETSC_DEVICE_HIP,
    121:   PETSC_DEVICE_SYCL,
    122:   PETSC_DEVICE_MAX
    123: } PetscDeviceType;
    124: PETSC_EXTERN const char *const PetscDeviceTypes[];
    125: #if defined(PETSC_HAVE_HIP)
    126: #  define PETSC_DEVICE_DEFAULT PETSC_DEVICE_HIP
    127: #elif defined(PETSC_HAVE_CUDA)
    128: #  define PETSC_DEVICE_DEFAULT PETSC_DEVICE_CUDA
    129: #elif PetscDefined(HAVE_SYCL)
    130: #  define PETSC_DEVICE_DEFAULT PETSC_DEVICE_SYCL
    131: #else
    132: #  define PETSC_DEVICE_DEFAULT PETSC_DEVICE_INVALID
    133: #endif

    135: /*S
    136:   PetscDevice - Handle to an accelerator "device" (usually a GPU)

    138:   Notes:
    139:   This object is used to house configuration and state of a device, but does not offer any ability to interact with or
    140:   drive device computation. This functionality is facilitated instead by the PetscDeviceContext object.

    142:   Level: beginner

    144: .seealso: PetscDeviceType, PetscDeviceInitType, PetscDeviceCreate(), PetscDeviceConfigure(), PetscDeviceDestroy(), PetscDeviceContext, PetscDeviceContextSetDevice(), PetscDeviceContextGetDevice()
    145: S*/
    146: typedef struct _n_PetscDevice *PetscDevice;

    148: /*E
    149:   PetscStreamType - Stream blocking mode, indicates how a stream implementation will interact with the default "NULL"
    150:   stream, which is usually blocking.

    152: $ PETSC_STREAM_GLOBAL_BLOCKING    - Alias for NULL stream. Any stream of this type will block the host for all other streams to finish work before starting its operations.
    153: $ PETSC_STREAM_DEFAULT_BLOCKING   - Stream will act independent of other streams, but will still be blocked by actions on the NULL stream.
    154: $ PETSC_STREAM_GLOBAL_NONBLOCKING - Stream is truly asynchronous, and is blocked by nothing, not even the NULL stream.
    155: $ PETSC_STREAM_MAX                - Always 1 greater than the largest PetscStreamType, do not use

    157:   Level: intermediate

    159: .seealso: PetscDeviceContextSetStreamType(), PetscDeviceContextGetStreamType()
    160: E*/
    161: typedef enum {
    162:   PETSC_STREAM_GLOBAL_BLOCKING,
    163:   PETSC_STREAM_DEFAULT_BLOCKING,
    164:   PETSC_STREAM_GLOBAL_NONBLOCKING,
    165:   PETSC_STREAM_MAX
    166: } PetscStreamType;
    167: PETSC_EXTERN const char *const PetscStreamTypes[];

    169: /*E
    170:   PetscDeviceContextJoinMode - Describes the type of join operation to perform in PetscDeviceContextJoin()

    172: $ PETSC_DEVICE_CONTEXT_DESTROY - Destroy all incoming sub-contexts after join.
    173: $ PETSC_CONTEXT_JOIN_SYNC      - Synchronize incoming sub-contexts after join.
    174: $ PETSC_CONTEXT_JOIN_NO_SYNC   - Do not synchronize incoming sub-contexts after join.

    176:   Level: beginner

    178: .seealso: PetscDeviceContext, PetscDeviceContextFork(), PetscDeviceContextJoin()
    179: E*/
    180: typedef enum {
    181:   PETSC_DEVICE_CONTEXT_JOIN_DESTROY,
    182:   PETSC_DEVICE_CONTEXT_JOIN_SYNC,
    183:   PETSC_DEVICE_CONTEXT_JOIN_NO_SYNC
    184: } PetscDeviceContextJoinMode;
    185: PETSC_EXTERN const char *const PetscDeviceContextJoinModes[];

    187: /*S
    188:   PetscDeviceContext - Container to manage stream dependencies and the various solver handles for asynchronous device compute.

    190:   Level: beginner

    192: .seealso: PetscDevice, PetscDeviceContextCreate(), PetscDeviceContextSetDevice(), PetscDeviceContextDestroy(),
    193: PetscDeviceContextFork(), PetscDeviceContextJoin()
    194: S*/
    195: typedef struct _n_PetscDeviceContext *PetscDeviceContext;
    196: #endif /* PETSCDEVICETYPES_H */