Actual source code: sycldevice.sycl.cxx

  1: #include "../../interface/sycldevice.hpp"
  2: #include <csetjmp> // for MPI sycl device awareness
  3: #include <csignal> // SIGSEGV
  4: #include <vector>
  5: #include <CL/sycl.hpp>

  7: #if PetscDefined(USE_LOG)
  8:   PETSC_INTERN PetscErrorCode PetscLogInitialize(void);
  9: #else
 10:   #define PetscLogInitialize() 0
 11: #endif

 13: namespace Petsc
 14: {

 16: namespace Device
 17: {

 19: namespace SYCL
 20: {

 22: // definition for static
 23: std::array<Device::DeviceInternal*,PETSC_DEVICE_MAX_DEVICES> Device::devices_array_ = {};
 24: Device::DeviceInternal**                                     Device::devices_       = &Device::devices_array_[1];
 25: int                                                          Device::defaultDevice_ = PETSC_SYCL_DEVICE_NONE;
 26: bool                                                         Device::initialized_   = false;

 28: static std::jmp_buf MPISyclAwareJumpBuffer;
 29: static bool         MPISyclAwareJumpBufferSet;

 31: // internal "impls" class for SyclDevice. Each instance represents a single sycl device
 32: class PETSC_NODISCARD Device::DeviceInternal
 33: {
 34:   const int            id_; // -1 for the host device; 0 and up for gpu devices
 35:   bool                 devInitialized_;
 36:   const sycl::device   syclDevice_;

 38: public:
 39:   // default constructor
 40:   DeviceInternal(int id) noexcept : id_(id),devInitialized_(false),syclDevice_(chooseSYCLDevice_(id)){}
 41:   int  id() const {return id_;}
 42:   bool initialized() const {return devInitialized_;}

 44:   PETSC_NODISCARD PetscErrorCode initialize() noexcept
 45:   {
 46:     if (devInitialized_) return 0;
 47:     if (syclDevice_.is_gpu() && use_gpu_aware_mpi) {
 48:       if (!isMPISyclAware_()) {
 49:         (*PetscErrorPrintf)("PETSc is configured with sycl support, but your MPI is not aware of sycl GPU devices. For better performance, please use a sycl GPU-aware MPI.\n");
 50:         (*PetscErrorPrintf)("If you do not care, add option -use_gpu_aware_mpi 0. To not see the message again, add the option to your .petscrc, OR add it to the env var PETSC_OPTIONS.\n");
 51:         PETSCABORT(PETSC_COMM_SELF,PETSC_ERR_LIB);
 52:       }
 53:     }
 54:     devInitialized_ = true;
 55:     return 0;
 56:   }

 58:   PETSC_NODISCARD PetscErrorCode view(PetscViewer viewer) const noexcept
 59:   {
 60:     MPI_Comm       comm;
 61:     PetscMPIInt    rank;
 62:     PetscBool      iascii;

 65:     PetscObjectTypeCompare(reinterpret_cast<PetscObject>(viewer),PETSCVIEWERASCII,&iascii);
 66:     PetscObjectGetComm(reinterpret_cast<PetscObject>(viewer),&comm);
 67:     if (iascii) {
 68:       PetscViewer sviewer;

 70:       MPI_Comm_rank(comm,&rank);
 71:       PetscViewerGetSubViewer(viewer,PETSC_COMM_SELF,&sviewer);
 72:       PetscViewerASCIIPrintf(sviewer,"[%d] device: %s\n",rank,syclDevice_.get_info<sycl::info::device::name>().c_str());
 73:       PetscViewerASCIIPushTab(sviewer);
 74:       PetscViewerASCIIPrintf(sviewer,"-> Device vendor: %s\n",syclDevice_.get_info<sycl::info::device::vendor>().c_str());
 75:       PetscViewerASCIIPopTab(sviewer);
 76:       PetscViewerFlush(sviewer);
 77:       PetscViewerRestoreSubViewer(viewer,PETSC_COMM_SELF,&sviewer);
 78:       PetscViewerFlush(viewer);
 79:     }
 80:     return 0;
 81:   }

 83: private:
 84:   static sycl::device chooseSYCLDevice_(int id)
 85:   {
 86:     if (id == PETSC_SYCL_DEVICE_HOST) {
 87:       return sycl::device(sycl::host_selector());
 88:     } else {
 89:       return sycl::device::get_devices(sycl::info::device_type::gpu)[id];
 90:     }
 91:   }

 93:   // Is the underlying MPI aware of sycl (GPU) devices?
 94:   bool isMPISyclAware_() noexcept
 95:   {
 96:     const int      bufSize = 2;
 97:     const int      hbuf[bufSize] = {1,0};
 98:     int            *dbuf = nullptr;
 99:     bool           awareness = false;
100:     const auto     SyclSignalHandler = [](int signal, void *ptr) -> PetscErrorCode {
101:       if ((signal == SIGSEGV) && MPISyclAwareJumpBufferSet) std::longjmp(MPISyclAwareJumpBuffer,1);
102:       return PetscSignalHandlerDefault(signal,ptr);
103:     };

105:     auto Q = sycl::queue(syclDevice_);
106:     dbuf   = sycl::malloc_device<int>(bufSize,Q);
107:     Q.memcpy(dbuf,hbuf,sizeof(int)*bufSize).wait();
108:     PETSC_COMM_SELF,PetscPushSignalHandler(SyclSignalHandler,nullptr);
109:     MPISyclAwareJumpBufferSet = true;
110:     if (setjmp(MPISyclAwareJumpBuffer)) {
111:       // if a segv was triggered in the MPI_Allreduce below, it is very likely due to MPI not being GPU-aware
112:       awareness = false;
113:       PetscStackPop;
114:     } else if (!MPI_Allreduce(dbuf,dbuf+1,1,MPI_INT,MPI_SUM,PETSC_COMM_SELF)) awareness = true;
115:     MPISyclAwareJumpBufferSet = false;
116:     PETSC_COMM_SELF,PetscPopSignalHandler();
117:     sycl::free(dbuf,Q);
118:     return awareness;
119:   }
120: };

122: PetscErrorCode Device::initialize(MPI_Comm comm, PetscInt *defaultDeviceId, PetscDeviceInitType *defaultInitType) noexcept
123: {
124:   PetscInt       initType = *defaultInitType,id = *defaultDeviceId;
125:   PetscBool      view = PETSC_FALSE,flg;
126:   PetscInt       ngpus;

129:   if (initialized_) return 0;
130:   initialized_ = true;
131:   PetscRegisterFinalize(finalize_);

133:   PetscOptionsBegin(comm,nullptr,"PetscDevice SYCL Options","Sys");
134:   PetscOptionsEList("-device_enable_sycl","How (or whether) to initialize a device","SyclDevice::initialize()",PetscDeviceInitTypes,3,PetscDeviceInitTypes[initType],&initType,nullptr);
135:   PetscOptionsRangeInt("-device_select_sycl","Which sycl device to use? Pass -2 for host, PETSC_DECIDE (-1) to let PETSc decide, 0 and up for GPUs","PetscDeviceCreate",id,&id,nullptr,-2,std::numeric_limits<decltype(ngpus)>::max());
136:   PetscOptionsBool("-device_view_sycl","Display device information and assignments (forces eager initialization)",nullptr,view,&view,&flg);
137:   PetscOptionsEnd();

139:   // post-process the options and lay the groundwork for initialization if needs be
140:   std::vector<sycl::device> gpu_devices = sycl::device::get_devices(sycl::info::device_type::gpu);
141:   ngpus = static_cast<PetscInt>(gpu_devices.size());

145:   if (initType == PETSC_DEVICE_INIT_NONE) id = PETSC_SYCL_DEVICE_NONE; /* user wants to disable all sycl devices */
146:   else {
147:     PetscDeviceCheckDeviceCount_Internal(ngpus);
148:     if (id == PETSC_DECIDE) { /* petsc will choose a GPU device if any, otherwise a CPU device */
149:       if (ngpus) {
150:         PetscMPIInt rank;
151:         MPI_Comm_rank(comm,&rank);
152:         id   = rank % ngpus;
153:       } else id = PETSC_SYCL_DEVICE_HOST;
154:     }
155:     view = static_cast<decltype(view)>(view && flg);
156:     if (view) initType = PETSC_DEVICE_INIT_EAGER;
157:   }

159:   if (id == -2) id = PETSC_SYCL_DEVICE_HOST; // user passed in '-device_select_sycl -2'. We transform it into canonical form

161:   defaultDevice_ = static_cast<decltype(defaultDevice_)>(id);

164:   if (initType == PETSC_DEVICE_INIT_EAGER) {
165:     devices_[defaultDevice_] = new DeviceInternal(defaultDevice_);
166:     devices_[defaultDevice_]->initialize();
167:     if (view) {
168:       PetscViewer viewer;
169:       PetscLogInitialize();
170:       PetscViewerASCIIGetStdout(comm,&viewer);
171:       devices_[defaultDevice_]->view(viewer);
172:     }
173:   }

175:   // record the results of the initialization
176:   *defaultInitType = static_cast<PetscDeviceInitType>(initType);
177:   *defaultDeviceId = id;
178:   return 0;
179: }

181: PetscErrorCode Device::finalize_() noexcept
182: {
183:   if (!initialized_) return 0;
184:   for (auto&& devPtr : devices_array_) delete devPtr;
185:   defaultDevice_ = PETSC_SYCL_DEVICE_NONE;  // disabled by default
186:   initialized_   = false;
187:   return 0;
188: }

190: PetscErrorCode Device::getDevice(PetscDevice device, PetscInt id) const noexcept
191: {
193:   if (id == PETSC_DECIDE) id = defaultDevice_;
195:   if (devices_[id]) {
197:   } else devices_[id] = new DeviceInternal(id);
198:   devices_[id]->initialize();
199:   device->deviceId           = devices_[id]->id(); // technically id = devices_[id]->id_ here
200:   device->ops->createcontext = create_;
201:   device->ops->configure     = this->configureDevice;
202:   device->ops->view          = this->viewDevice;
203:   return 0;
204: }

206: PetscErrorCode Device::configureDevice(PetscDevice device) noexcept
207: {
208:   // Nothing for now
209:   return 0;
210: }

212: PetscErrorCode Device::viewDevice(PetscDevice device, PetscViewer viewer) noexcept
213: {
214:   devices_[device->deviceId]->view(viewer);
215:   return 0;
216: }

218: } // namespace SYCL

220: } // namespace Device

222: } // namespace Petsc