Date: 2022-08-02T03:13:24+00:00

Actual source code: sycldevice.sycl.cxx

      1: #include "../../interface/sycldevice.hpp"
      2: #include <csetjmp> // for MPI sycl device awareness
      3: #include <csignal> // SIGSEGV
      4: #include <vector>
      5: #include <CL/sycl.hpp>

      7: #if PetscDefined(USE_LOG)
      8:   PETSC_INTERN PetscErrorCode PetscLogInitialize(void);
      9: #else
     10:   #define PetscLogInitialize() 0
     11: #endif

     13: namespace Petsc
     14: {

     16: namespace Device
     17: {

     19: namespace SYCL
     20: {

     22: // definition for static
     23: std::array<Device::DeviceInternal*,PETSC_DEVICE_MAX_DEVICES> Device::devices_array_ = {};
     24: Device::DeviceInternal**                                     Device::devices_       = &Device::devices_array_[1];
     25: int                                                          Device::defaultDevice_ = PETSC_SYCL_DEVICE_NONE;
     26: bool                                                         Device::initialized_   = false;

     28: static std::jmp_buf MPISyclAwareJumpBuffer;
     29: static bool         MPISyclAwareJumpBufferSet;

     31: // internal "impls" class for SyclDevice. Each instance represents a single sycl device
     32: class PETSC_NODISCARD Device::DeviceInternal
     33: {
     34:   const int            id_; // -1 for the host device; 0 and up for gpu devices
     35:   bool                 devInitialized_;
     36:   const sycl::device   syclDevice_;

     38: public:
     39:   // default constructor
     40:   DeviceInternal(int id) noexcept : id_(id),devInitialized_(false),syclDevice_(chooseSYCLDevice_(id)){}
     41:   int  id() const {return id_;}
     42:   bool initialized() const {return devInitialized_;}

     44:   PETSC_NODISCARD PetscErrorCode initialize() noexcept
     45:   {
     46:     if (devInitialized_) return 0;
     47:     if (syclDevice_.is_gpu() && use_gpu_aware_mpi) {
     48:       if (!isMPISyclAware_()) {
     49:         (*PetscErrorPrintf)("PETSc is configured with sycl support, but your MPI is not aware of sycl GPU devices. For better performance, please use a sycl GPU-aware MPI.\n");
     50:         (*PetscErrorPrintf)("If you do not care, add option -use_gpu_aware_mpi 0. To not see the message again, add the option to your .petscrc, OR add it to the env var PETSC_OPTIONS.\n");
     51:         PETSCABORT(PETSC_COMM_SELF,PETSC_ERR_LIB);
     52:       }
     53:     }
     54:     devInitialized_ = true;
     55:     return 0;
     56:   }

     58:   PETSC_NODISCARD PetscErrorCode view(PetscViewer viewer) const noexcept
     59:   {
     60:     MPI_Comm       comm;
     61:     PetscMPIInt    rank;
     62:     PetscBool      iascii;

     65:     PetscObjectTypeCompare(reinterpret_cast<PetscObject>(viewer),PETSCVIEWERASCII,&iascii);
     66:     PetscObjectGetComm(reinterpret_cast<PetscObject>(viewer),&comm);
     67:     if (iascii) {
     68:       PetscViewer sviewer;

     70:       MPI_Comm_rank(comm,&rank);
     71:       PetscViewerGetSubViewer(viewer,PETSC_COMM_SELF,&sviewer);
     72:       PetscViewerASCIIPrintf(sviewer,"[%d] device: %s\n",rank,syclDevice_.get_info<sycl::info::device::name>().c_str());
     73:       PetscViewerASCIIPushTab(sviewer);
     74:       PetscViewerASCIIPrintf(sviewer,"-> Device vendor: %s\n",syclDevice_.get_info<sycl::info::device::vendor>().c_str());
     75:       PetscViewerASCIIPopTab(sviewer);
     76:       PetscViewerFlush(sviewer);
     77:       PetscViewerRestoreSubViewer(viewer,PETSC_COMM_SELF,&sviewer);
     78:       PetscViewerFlush(viewer);
     79:     }
     80:     return 0;
     81:   }

     83: private:
     84:   static sycl::device chooseSYCLDevice_(int id)
     85:   {
     86:     if (id == PETSC_SYCL_DEVICE_HOST) {
     87:       return sycl::device(sycl::host_selector());
     88:     } else {
     89:       return sycl::device::get_devices(sycl::info::device_type::gpu)[id];
     90:     }
     91:   }

     93:   // Is the underlying MPI aware of sycl (GPU) devices?
     94:   bool isMPISyclAware_() noexcept
     95:   {
     96:     const int      bufSize = 2;
     97:     const int      hbuf[bufSize] = {1,0};
     98:     int            *dbuf = nullptr;
     99:     bool           awareness = false;
    100:     const auto     SyclSignalHandler = [](int signal, void *ptr) -> PetscErrorCode {
    101:       if ((signal == SIGSEGV) && MPISyclAwareJumpBufferSet) std::longjmp(MPISyclAwareJumpBuffer,1);
    102:       return PetscSignalHandlerDefault(signal,ptr);
    103:     };

    105:     auto Q = sycl::queue(syclDevice_);
    106:     dbuf   = sycl::malloc_device<int>(bufSize,Q);
    107:     Q.memcpy(dbuf,hbuf,sizeof(int)*bufSize).wait();
    108:     PETSC_COMM_SELF,PetscPushSignalHandler(SyclSignalHandler,nullptr);
    109:     MPISyclAwareJumpBufferSet = true;
    110:     if (setjmp(MPISyclAwareJumpBuffer)) {
    111:       // if a segv was triggered in the MPI_Allreduce below, it is very likely due to MPI not being GPU-aware
    112:       awareness = false;
    113:       PetscStackPop;
    114:     } else if (!MPI_Allreduce(dbuf,dbuf+1,1,MPI_INT,MPI_SUM,PETSC_COMM_SELF)) awareness = true;
    115:     MPISyclAwareJumpBufferSet = false;
    116:     PETSC_COMM_SELF,PetscPopSignalHandler();
    117:     sycl::free(dbuf,Q);
    118:     return awareness;
    119:   }
    120: };

    122: PetscErrorCode Device::initialize(MPI_Comm comm, PetscInt *defaultDeviceId, PetscDeviceInitType *defaultInitType) noexcept
    123: {
    124:   PetscInt       initType = *defaultInitType,id = *defaultDeviceId;
    125:   PetscBool      view = PETSC_FALSE,flg;
    126:   PetscInt       ngpus;

    129:   if (initialized_) return 0;
    130:   initialized_ = true;
    131:   PetscRegisterFinalize(finalize_);

    133:   PetscOptionsBegin(comm,nullptr,"PetscDevice SYCL Options","Sys");
    134:   PetscOptionsEList("-device_enable_sycl","How (or whether) to initialize a device","SyclDevice::initialize()",PetscDeviceInitTypes,3,PetscDeviceInitTypes[initType],&initType,nullptr);
    135:   PetscOptionsRangeInt("-device_select_sycl","Which sycl device to use? Pass -2 for host, PETSC_DECIDE (-1) to let PETSc decide, 0 and up for GPUs","PetscDeviceCreate",id,&id,nullptr,-2,std::numeric_limits<decltype(ngpus)>::max());
    136:   PetscOptionsBool("-device_view_sycl","Display device information and assignments (forces eager initialization)",nullptr,view,&view,&flg);
    137:   PetscOptionsEnd();

    139:   // post-process the options and lay the groundwork for initialization if needs be
    140:   std::vector<sycl::device> gpu_devices = sycl::device::get_devices(sycl::info::device_type::gpu);
    141:   ngpus = static_cast<PetscInt>(gpu_devices.size());

    145:   if (initType == PETSC_DEVICE_INIT_NONE) id = PETSC_SYCL_DEVICE_NONE; /* user wants to disable all sycl devices */
    146:   else {
    147:     PetscDeviceCheckDeviceCount_Internal(ngpus);
    148:     if (id == PETSC_DECIDE) { /* petsc will choose a GPU device if any, otherwise a CPU device */
    149:       if (ngpus) {
    150:         PetscMPIInt rank;
    151:         MPI_Comm_rank(comm,&rank);
    152:         id   = rank % ngpus;
    153:       } else id = PETSC_SYCL_DEVICE_HOST;
    154:     }
    155:     view = static_cast<decltype(view)>(view && flg);
    156:     if (view) initType = PETSC_DEVICE_INIT_EAGER;
    157:   }

    159:   if (id == -2) id = PETSC_SYCL_DEVICE_HOST; // user passed in '-device_select_sycl -2'. We transform it into canonical form

    161:   defaultDevice_ = static_cast<decltype(defaultDevice_)>(id);

    164:   if (initType == PETSC_DEVICE_INIT_EAGER) {
    165:     devices_[defaultDevice_] = new DeviceInternal(defaultDevice_);
    166:     devices_[defaultDevice_]->initialize();
    167:     if (view) {
    168:       PetscViewer viewer;
    169:       PetscLogInitialize();
    170:       PetscViewerASCIIGetStdout(comm,&viewer);
    171:       devices_[defaultDevice_]->view(viewer);
    172:     }
    173:   }

    175:   // record the results of the initialization
    176:   *defaultInitType = static_cast<PetscDeviceInitType>(initType);
    177:   *defaultDeviceId = id;
    178:   return 0;
    179: }

    181: PetscErrorCode Device::finalize_() noexcept
    182: {
    183:   if (!initialized_) return 0;
    184:   for (auto&& devPtr : devices_array_) delete devPtr;
    185:   defaultDevice_ = PETSC_SYCL_DEVICE_NONE;  // disabled by default
    186:   initialized_   = false;
    187:   return 0;
    188: }

    190: PetscErrorCode Device::getDevice(PetscDevice device, PetscInt id) const noexcept
    191: {
    193:   if (id == PETSC_DECIDE) id = defaultDevice_;
    195:   if (devices_[id]) {
    197:   } else devices_[id] = new DeviceInternal(id);
    198:   devices_[id]->initialize();
    199:   device->deviceId           = devices_[id]->id(); // technically id = devices_[id]->id_ here
    200:   device->ops->createcontext = create_;
    201:   device->ops->configure     = this->configureDevice;
    202:   device->ops->view          = this->viewDevice;
    203:   return 0;
    204: }

    206: PetscErrorCode Device::configureDevice(PetscDevice device) noexcept
    207: {
    208:   // Nothing for now
    209:   return 0;
    210: }

    212: PetscErrorCode Device::viewDevice(PetscDevice device, PetscViewer viewer) noexcept
    213: {
    214:   devices_[device->deviceId]->view(viewer);
    215:   return 0;
    216: }

    218: } // namespace SYCL

    220: } // namespace Device

    222: } // namespace Petsc