Date: 2022-03-04T17:37:10+00:00

Actual source code: dcontext.cxx

      1: #include <petsc/private/deviceimpl.h>
      2: #include "objpool.hpp"

      4: /* Define the allocator */
      5: struct PetscDeviceContextAllocator : Petsc::Allocator<PetscDeviceContext>
      6: {
      7:   static PetscInt PetscDeviceContextID;

      9:   PETSC_NODISCARD PetscErrorCode create(PetscDeviceContext *dctx) noexcept
     10:   {
     11:     PetscDeviceContext dc;
     12:     PetscErrorCode     ierr;

     15:     PetscNew(&dc);
     16:     dc->id         = PetscDeviceContextID++;
     17:     dc->idle       = PETSC_TRUE;
     18:     dc->streamType = PETSC_STREAM_DEFAULT_BLOCKING;
     19:     *dctx          = dc;
     20:     return(0);
     21:   }

     23:   PETSC_NODISCARD PetscErrorCode destroy(PetscDeviceContext &dctx) const noexcept
     24:   {

     28:     if (PetscUnlikelyDebug(dctx->numChildren)) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_ARG_WRONGSTATE,"Device context still has %D un-restored children, must call PetscDeviceContextRestore() on all children before destroying",dctx->numChildren);
     29:     if (dctx->ops->destroy) {(*dctx->ops->destroy)(dctx);}
     30:     PetscDeviceDestroy(&dctx->device);
     31:     PetscFree(dctx->childIDs);
     32:     PetscFree(dctx);
     33:     return(0);
     34:   }

     36:   PETSC_NODISCARD PetscErrorCode reset(PetscDeviceContext &dctx) const noexcept
     37:   {

     41:     /* don't deallocate the child array, rather just zero it out */
     42:     PetscArrayzero(dctx->childIDs,dctx->maxNumChildren);
     43:     dctx->setup       = PETSC_FALSE;
     44:     dctx->numChildren = 0;
     45:     dctx->idle        = PETSC_TRUE;
     46:     dctx->streamType  = PETSC_STREAM_DEFAULT_BLOCKING;
     47:     return(0);
     48:   }

     50:   PETSC_NODISCARD PetscErrorCode finalize(void) noexcept
     51:   {
     53:     PetscDeviceContextID = 0;
     54:     return(0);
     55:   }
     56: };
     57: PetscInt PetscDeviceContextAllocator::PetscDeviceContextID = 0;

     59: static Petsc::ObjectPool<PetscDeviceContext,PetscDeviceContextAllocator> contextPool;

     61: /*@C
     62:   PetscDeviceContextCreate - Creates a PetscDeviceContext

     64:   Not Collective, Asynchronous

     66:   Output Paramemter:
     67: . dctx - The PetscDeviceContext

     69:   Notes:
     70:   Unlike almost every other PETSc class it is advised that most users use
     71:   PetscDeviceContextDuplicate() rather than this routine to create new contexts. Contexts
     72:   of different types are incompatible with one another; using
     73:   PetscDeviceContextDuplicate() ensures compatible types.

     75:   Level: beginner

     77: .seealso: PetscDeviceContextDuplicate(), PetscDeviceContextSetDevice(),
     78: PetscDeviceContextSetStreamType(), PetscDeviceContextSetUp(),
     79: PetscDeviceContextSetFromOptions(), PetscDeviceContextDestroy()
     80: @*/
     81: PetscErrorCode PetscDeviceContextCreate(PetscDeviceContext *dctx)
     82: {

     87:   PetscDeviceInitializePackage();
     88:   contextPool.get(*dctx);
     89:   return(0);
     90: }

     92: /*@C
     93:   PetscDeviceContextDestroy - Frees a PetscDeviceContext

     95:   Not Collective, Asynchronous

     97:   Input Parameters:
     98: . dctx - The PetscDeviceContext

    100:   Notes:
    101:   No implicit synchronization occurs due to this routine, all resources are released completely asynchronously
    102:   w.r.t. the host. If one needs to guarantee access to the data produced on this contexts stream one should perform the
    103:   appropriate synchronization before calling this routine.

    105:   Developer Notes:
    106:   The context is never actually "destroyed", only returned to an ever growing pool of
    107:   contexts. There are currently no safeguards on the size of the pool, this should perhaps
    108:   be implemented.

    110:   Level: beginner

    112: .seealso: PetscDeviceContextCreate(), PetscDeviceContextSetDevice(), PetscDeviceContextSetUp(), PetscDeviceContextSynchronize()
    113: @*/
    114: PetscErrorCode PetscDeviceContextDestroy(PetscDeviceContext *dctx)
    115: {

    119:   if (!*dctx) return(0);
    120:   /* use move assignment whenever possible */
    121:   contextPool.reclaim(std::move(*dctx));
    122:   return(0);
    123: }

    125: /*@C
    126:   PetscDeviceContextSetStreamType - Set the implementation type of the underlying stream for a PetscDeviceContext

    128:   Not Collective, Asynchronous

    130:   Input Parameters:
    131: + dctx - The PetscDeviceContext
    132: - type - The PetscStreamType

    134:   Notes:
    135:   See PetscStreamType in include/petscdevicetypes.h for more information on the available
    136:   types and their interactions. If the PetscDeviceContext was previously set up and stream
    137:   type was changed, you must call PetscDeviceContextSetUp() again after this routine.

    139:   Level: intermediate

    141: .seealso: PetscDeviceContextGetStreamType(), PetscDeviceContextCreate(), PetscDeviceContextSetUp(), PetscDeviceContextSetFromOptions()
    142: @*/
    143: PetscErrorCode PetscDeviceContextSetStreamType(PetscDeviceContext dctx, PetscStreamType type)
    144: {
    148:   /* only need to do complex swapping if the object has already been setup */
    149:   if (dctx->setup && (dctx->streamType != type)) {

    152:     (*dctx->ops->changestreamtype)(dctx,type);
    153:     dctx->setup = PETSC_FALSE;
    154:   }
    155:   dctx->streamType = type;
    156:   return(0);
    157: }

    159: /*@C
    160:   PetscDeviceContextGetStreamType - Get the implementation type of the underlying stream for a PetscDeviceContext

    162:   Not Collective, Asynchronous

    164:   Input Parameter:
    165: . dctx - The PetscDeviceContext

    167:   Output Parameter:
    168: . type - The PetscStreamType

    170:   Notes:
    171:   See PetscStreamType in include/petscdevicetypes.h for more information on the available types and their interactions

    173:   Level: intermediate

    175: .seealso: PetscDeviceContextSetStreamType(), PetscDeviceContextCreate(), PetscDeviceContextSetFromOptions()
    176: @*/
    177: PetscErrorCode PetscDeviceContextGetStreamType(PetscDeviceContext dctx, PetscStreamType *type)
    178: {
    182:   *type = dctx->streamType;
    183:   return(0);
    184: }

    186: /*@C
    187:   PetscDeviceContextSetDevice - Set the underlying device for the PetscDeviceContext

    189:   Not Collective, Possibly Synchronous

    191:   Input Parameters:
    192: + dctx   - The PetscDeviceContext
    193: - device - The PetscDevice

    195:   Notes:
    196:   This routine is effectively PetscDeviceContext's "set-type" (so every PetscDeviceContext
    197:   must also have an attached PetscDevice). Unlike the usual set-type semantics, it is
    198:   not stricly necessary to set a contexts device to enable usage, any created device
    199:   contexts will always come equipped with the "default" device.

    201:   This routine may initialize the backend device and incur synchronization.

    203:   Level: intermediate

    205: .seealso: PetscDeviceCreate(), PetscDeviceConfigure(), PetscDeviceContextGetDevice()
    206: @*/
    207: PetscErrorCode PetscDeviceContextSetDevice(PetscDeviceContext dctx, PetscDevice device)
    208: {

    214:   if (dctx->device == device) return(0);
    215:   PetscDeviceDestroy(&dctx->device);
    216:   PetscMemzero(dctx->ops,sizeof(*dctx->ops));
    217:   (*device->ops->createcontext)(dctx);
    218:   dctx->device = PetscDeviceReference(device);
    219:   dctx->setup  = PETSC_FALSE;
    220:   return(0);
    221: }

    223: /*@C
    224:   PetscDeviceContextGetDevice - Get the underlying PetscDevice for a PetscDeviceContext

    226:   Not Collective, Asynchronous

    228:   Input Parameter:
    229: . dctx - the PetscDeviceContext

    231:   Output Parameter:
    232: . device - The PetscDevice

    234:   Notes:
    235:   This is a borrowed reference, the user should not destroy the device.

    237:   Level: intermediate

    239: .seealso: PetscDeviceContextSetDevice(), PetscDevice
    240: @*/
    241: PetscErrorCode PetscDeviceContextGetDevice(PetscDeviceContext dctx, PetscDevice *device)
    242: {
    246:   *device = dctx->device;
    247:   return(0);
    248: }

    250: /*@C
    251:   PetscDeviceContextSetUp - Prepares a PetscDeviceContext for use

    253:   Not Collective, Asynchronous

    255:   Input Parameter:
    256: . dctx - The PetscDeviceContext

    258:   Developer Notes:
    259:   This routine is usually the stage where a PetscDeviceContext acquires device-side data structures such as streams,
    260:   events, and (possibly) handles.

    262:   Level: beginner

    264: .seealso: PetscDeviceContextCreate(), PetscDeviceContextSetDevice(), PetscDeviceContextDestroy(), PetscDeviceContextSetFromOptions()
    265: @*/
    266: PetscErrorCode PetscDeviceContextSetUp(PetscDeviceContext dctx)
    267: {

    272:   if (!dctx->device) {
    273:     PetscInfo2(NULL,"PetscDeviceContext %d did not have an explicitly attached PetscDevice, using default with type %s\n",dctx->id,PetscDeviceKinds[PETSC_DEVICE_DEFAULT]);
    274:     PetscDeviceContextSetDevice(dctx,PetscDeviceDefault_Internal());
    275:   }
    276:   if (dctx->setup) return(0);
    277:   (*dctx->ops->setup)(dctx);
    278:   dctx->setup = PETSC_TRUE;
    279:   return(0);
    280: }

    282: /*@C
    283:   PetscDeviceContextDuplicate - Duplicates a PetscDeviceContext object

    285:   Not Collective, Asynchronous

    287:   Input Parameter:
    288: . dctx - The PetscDeviceContext to duplicate

    290:   Output Paramter:
    291: . strmdup - The duplicated PetscDeviceContext

    293:   Notes:
    294:   This is a shorthand method for creating a PetscDeviceContext with the exact same
    295:   settings as another. Note however that the duplicated PetscDeviceContext does not "share"
    296:   any of the underlying data with the original, (including its current stream-state) they
    297:   are completely separate objects.

    299:   Level: beginner

    301: .seealso: PetscDeviceContextCreate(), PetscDeviceContextSetDevice(), PetscDeviceContextSetStreamType()
    302: @*/
    303: PetscErrorCode PetscDeviceContextDuplicate(PetscDeviceContext dctx, PetscDeviceContext *dctxdup)
    304: {

    310:   PetscDeviceContextCreate(dctxdup);
    311:   PetscDeviceContextSetDevice(*dctxdup,dctx->device);
    312:   PetscDeviceContextSetStreamType(*dctxdup,dctx->streamType);
    313:   PetscDeviceContextSetUp(*dctxdup);
    314:   return(0);
    315: }

    317: /*@C
    318:   PetscDeviceContextQueryIdle - Returns whether or not a PetscDeviceContext is idle

    320:   Not Collective, Asynchronous

    322:   Input Parameter:
    323: . dctx - The PetscDeviceContext object

    325:   Output Parameter:
    326: . idle - PETSC_TRUE if PetscDeviceContext has NO work, PETSC_FALSE if it has work

    328:   Notes:
    329:   This routine only refers a singular context and does NOT take any of its children into account. That is, if dctx is
    330:   idle but has dependents who do have work, this routine still returns PETSC_TRUE.

    332:   Results of PetscDeviceContextQueryIdle() are cached on return, allowing this function to be called repeatedly in an
    333:   efficient manner. When debug mode is enabled this cache is verified on every call to
    334:   this routine, but is blindly believed when debugging is disabled.

    336:   Level: intermediate

    338: .seealso: PetscDeviceContextCreate(), PetscDeviceContextWaitForContext(), PetscDeviceContextFork()
    339: @*/
    340: PetscErrorCode PetscDeviceContextQueryIdle(PetscDeviceContext dctx, PetscBool *idle)
    341: {

    347:   if (dctx->idle) {
    348:     *idle = PETSC_TRUE;
    349:     PetscDeviceContextValidateIdle_Internal(dctx);
    350:   } else {
    351:     (*dctx->ops->query)(dctx,idle);
    352:     dctx->idle = *idle;
    353:   }
    354:   return(0);
    355: }

    357: /*@C
    358:   PetscDeviceContextWaitForContext - Make one context wait for another context to finish

    360:   Not Collective, Asynchronous

    362:   Input Parameters:
    363: + dctxa - The PetscDeviceContext object that is waiting
    364: - dctxb - The PetscDeviceContext object that is being waited on

    366:   Notes:
    367:   Serializes two PetscDeviceContexts. This routine uses only the state of dctxb at the moment this routine was
    368:   called, so any future work queued will not affect dctxa. It is safe to pass the same context to both arguments.

    370:   Level: beginner

    372: .seealso: PetscDeviceContextCreate(), PetscDeviceContextQueryIdle(), PetscDeviceContextJoin()
    373: @*/
    374: PetscErrorCode PetscDeviceContextWaitForContext(PetscDeviceContext dctxa, PetscDeviceContext dctxb)
    375: {

    380:   if (dctxa == dctxb) return(0);
    381:   if (dctxb->idle) {
    382:     /* No need to do the extra function lookup and event record if the stream were waiting on isn't doing anything */
    383:     PetscDeviceContextValidateIdle_Internal(dctxb);
    384:   } else {
    385:     (*dctxa->ops->waitforctx)(dctxa,dctxb);
    386:   }
    387:   return(0);
    388: }

    390: /*@C
    391:   PetscDeviceContextFork - Create a set of dependent child contexts from a parent context

    393:   Not Collective, Asynchronous

    395:   Input Parameters:
    396: + dctx - The parent PetscDeviceContext
    397: - n    - The number of children to create

    399:   Output Parameter:
    400: . dsub - The created child context(s)

    402:   Notes:
    403:   This routine creates n edges of a DAG from a source node which are causally dependent on the source node, meaning
    404:   that work queued on child contexts will not start until the parent context finishes its work. This accounts for work
    405:   queued on the parent up until calling this function, any subsequent work enqueued on the parent has no effect on the children.

    407:   Any children created with this routine have their lifetimes bounded by the parent. That is, the parent context expects
    408:   to free all of it's children (and ONLY its children) before itself is freed.

    410:   DAG representation:
    411: .vb
    412:   time ->

    414:   -> dctx \----> dctx ------>
    415:            \---> dsub[0] --->
    416:             \--> ... ------->
    417:              \-> dsub[n-1] ->
    418: .ve

    420:   Level: intermediate

    422: .seealso: PetscDeviceContextJoin(), PetscDeviceContextSynchronize(), PetscDeviceContextQueryIdle()
    423: @*/
    424: PetscErrorCode PetscDeviceContextFork(PetscDeviceContext dctx, PetscInt n, PetscDeviceContext **dsub)
    425: {
    426: #if defined(PETSC_USE_DEBUG) && defined(PETSC_USE_INFO)
    427:   const PetscInt      nBefore = n;
    428:   static std::string  idList;
    429: #endif
    430:   PetscDeviceContext *dsubTmp = nullptr;
    431:   PetscInt            i = 0;
    432:   PetscErrorCode      ierr;

    437:   if (PetscUnlikelyDebug(n < 0)) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_ARG_OUTOFRANGE,"Number of contexts requested %D < 0",n);
    438: #if defined(PETSC_USE_DEBUG) && defined(PETSC_USE_INFO)
    439:   /* reserve 4 chars per id, 2 for number and 2 for ', ' separator */
    440:   idList.reserve(4*n);
    441: #endif
    442:   /* update child totals */
    443:   dctx->numChildren += n;
    444:   /* now to find out if we have room */
    445:   if (dctx->numChildren > dctx->maxNumChildren) {
    446:     /* no room, either from having too many kids or not having any */
    447:     if (dctx->childIDs) {
    448:       /* have existing children, must reallocate them */
    449:       PetscRealloc(dctx->numChildren*sizeof(*dctx->childIDs),&dctx->childIDs);
    450:       /* clear the extra memory since realloc doesn't do it for us */
    451:       PetscArrayzero((dctx->childIDs)+(dctx->maxNumChildren),(dctx->numChildren)-(dctx->maxNumChildren));
    452:     } else {
    453:       /* have no children */
    454:       PetscCalloc1(dctx->numChildren,&dctx->childIDs);
    455:     }
    456:     /* update total number of children */
    457:     dctx->maxNumChildren = dctx->numChildren;
    458:   }
    459:   PetscMalloc1(n,&dsubTmp);
    460:   while (n) {
    461:     /* empty child slot */
    462:     if (!(dctx->childIDs[i])) {
    463:       /* create the child context in the image of its parent */
    464:       PetscDeviceContextDuplicate(dctx,dsubTmp+i);
    465:       PetscDeviceContextWaitForContext(dsubTmp[i],dctx);
    466:       /* register the child with its parent */
    467:       dctx->childIDs[i] = dsubTmp[i]->id;
    468: #if defined(PETSC_USE_DEBUG) && defined(PETSC_USE_INFO)
    469:       idList += std::to_string(dsubTmp[i]->id);
    470:       if (n != 1) idList += ", ";
    471: #endif
    472:       --n;
    473:     }
    474:     ++i;
    475:   }
    476: #if defined(PETSC_USE_DEBUG) && defined(PETSC_USE_INFO)
    477:   PetscInfo3(NULL,"Forked %D children from parent %D with IDs: %s\n",nBefore,dctx->id,idList.c_str());
    478:   /* resets the size but doesn't deallocate the memory */
    479:   idList.clear();
    480: #endif
    481:   /* pass the children back to caller */
    482:   *dsub = dsubTmp;
    483:   return(0);
    484: }

    486: /*@C
    487:   PetscDeviceContextJoin - Converge a set of child contexts

    489:   Not Collective, Asynchronous

    491:   Input Parameters:
    492: + dctx         - A PetscDeviceContext to converge on
    493: . n            - The number of sub contexts to converge
    494: . joinMode     - The type of join to perform
    495: - dsub         - The sub contexts to converge

    497:   Notes:
    498:   If PetscDeviceContextFork() creates n edges from a source node which all depend on the
    499:   source node, then this routine is the exact mirror. That is, it creates a node
    500:   (represented in dctx) which recieves n edges (and optionally destroys them) which is
    501:   dependent on the completion of all incoming edges.

    503:   If joinMode is PETSC_DEVICE_CONTEXT_JOIN_DESTROY all contexts in dsub will be destroyed
    504:   by this routine. Thus all sub contexts must have been created with the dctx passed to
    505:   this routine.

    507:   if joinMode is PETSC_DEVICE_CONTEXT_JOIN_NO_SYNC dctx waits for all sub contexts but the
    508:   sub contexts do not wait for one another afterwards.

    510:   If joinMode is PETSC_DEVICE_CONTEXT_JOIN_SYNC all sub contexts will additionally
    511:   wait on dctx after converging. This has the effect of "synchronizing" the outgoing
    512:   edges.

    514:   DAG representations:
    515:   If joinMode is PETSC_DEVICE_CONTEXT_JOIN_DESTROY
    516: .vb
    517:   time ->

    519:   -> dctx ---------/- dctx ->
    520:   -> dsub[0] -----/
    521:   ->  ... -------/
    522:   -> dsub[n-1] -/
    523: .ve
    524:   If joinMode is PETSC_DEVICE_CONTEXT_JOIN_NO_SYNC
    525: .vb
    526:   time ->

    528:   -> dctx ---------/- dctx ->
    529:   -> dsub[0] -----/--------->
    530:   ->  ... -------/---------->
    531:   -> dsub[n-1] -/----------->
    532: .ve
    533:   If joinMode is PETSC_DEVICE_CONTEXT_JOIN_SYNC
    534: .vb
    535:   time ->

    537:   -> dctx ---------/- dctx -\----> dctx ------>
    538:   -> dsub[0] -----/          \---> dsub[0] --->
    539:   ->  ... -------/            \--> ... ------->
    540:   -> dsub[n-1] -/              \-> dsub[n-1] ->
    541: .ve

    543:   Level: intermediate

    545: .seealso: PetscDeviceContextFork(), PetscDeviceContextSynchronize(), PetscDeviceContextJoinMode
    546: @*/
    547: PetscErrorCode PetscDeviceContextJoin(PetscDeviceContext dctx, PetscInt n, PetscDeviceContextJoinMode joinMode, PetscDeviceContext **dsub)
    548: {
    549: #if defined(PETSC_USE_DEBUG) && defined(PETSC_USE_INFO)
    550:   static std::string idList;
    551: #endif
    552:   PetscErrorCode     ierr;

    555:   /* validity of dctx is checked in the wait-for loop */
    557:   if (PetscUnlikelyDebug(n < 0)) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_ARG_OUTOFRANGE,"Number of contexts merged %D < 0",n);
    558: #if defined(PETSC_USE_DEBUG) && defined(PETSC_USE_INFO)
    559:   /* reserve 4 chars per id, 2 for number and 2 for ', ' separator */
    560:   idList.reserve(4*n);
    561: #endif
    562:   /* first dctx waits on all the incoming edges */
    563:   for (PetscInt i = 0; i < n; ++i) {
    565:     PetscDeviceContextWaitForContext(dctx,(*dsub)[i]);
    566: #if defined(PETSC_USE_DEBUG) && defined(PETSC_USE_INFO)
    567:     idList += std::to_string((*dsub)[i]->id);
    568:     if (i+1 < n) idList += ", ";
    569: #endif
    570:   }

    572:   /* now we handle the aftermath */
    573:   switch (joinMode) {
    574:   case PETSC_DEVICE_CONTEXT_JOIN_DESTROY:
    575:     {
    576:       PetscInt j = 0;

    578:       if (PetscUnlikelyDebug(n > dctx->numChildren)) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_ARG_OUTOFRANGE,"Trying to destroy %D children of a parent context that only has %D children, likely trying to restore to wrong parent",n,dctx->numChildren);
    579:       /* update child count while it's still fresh in memory */
    580:       dctx->numChildren -= n;
    581:       for (PetscInt i = 0; i < dctx->maxNumChildren; ++i) {
    582:         if (dctx->childIDs[i] && (dctx->childIDs[i] == (*dsub)[j]->id)) {
    583:           /* child is one of ours, can destroy it */
    584:           PetscDeviceContextDestroy((*dsub)+j);
    585:           /* reset the child slot */
    586:           dctx->childIDs[i] = 0;
    587:           if (++j == n) break;
    588:         }
    589:       }
    590:       /* gone through the loop but did not find every child, if this triggers (or well, doesn't) on perf-builds we leak the remaining contexts memory */
    591:       if (PetscUnlikelyDebug(j != n)) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_ARG_WRONG,"%D contexts still remain after destroy, this may be because you are trying to restore to the wrong parent context, or the device contexts are not in the same order as they were checked out out in.",n-j);
    592:       PetscFree(*dsub);
    593:     }
    594:     break;
    595:   case PETSC_DEVICE_CONTEXT_JOIN_SYNC:
    596:     for (PetscInt i = 0; i < n; ++i) {
    597:       PetscDeviceContextWaitForContext((*dsub)[i],dctx);
    598:     }
    599:   case PETSC_DEVICE_CONTEXT_JOIN_NO_SYNC:
    600:     break;
    601:   default:
    602:     SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_WRONG,"Unknown PetscDeviceContextJoinMode given");
    603:   }

    605: #if defined(PETSC_USE_DEBUG) && defined(PETSC_USE_INFO)
    606:   PetscInfo4(NULL,"Joined %D ctxs to ctx %D, mode %s with IDs: %s\n",n,dctx->id,PetscDeviceContextJoinModes[joinMode],idList.c_str());
    607:   idList.clear();
    608: #endif
    609:   return(0);
    610: }

    612: /*@C
    613:   PetscDeviceContextSynchronize - Block the host until all work queued on or associated with a PetscDeviceContext has finished

    615:   Not Collective, Synchronous

    617:   Input Parameters:
    618: . dctx - The PetscDeviceContext to synchronize

    620:   Level: beginner

    622: .seealso: PetscDeviceContextFork(), PetscDeviceContextJoin(), PetscDeviceContextQueryIdle()
    623: @*/
    624: PetscErrorCode PetscDeviceContextSynchronize(PetscDeviceContext dctx)
    625: {

    630:   /* if it isn't setup there is nothing to sync on */
    631:   if (dctx->setup) {(*dctx->ops->synchronize)(dctx);}
    632:   dctx->idle = PETSC_TRUE;
    633:   return(0);
    634: }

    636: static PetscDeviceContext globalContext      = nullptr;
    637: static PetscBool          globalContextSetup = PETSC_FALSE;
    638: static PetscStreamType    defaultStreamType  = PETSC_STREAM_DEFAULT_BLOCKING;

    640: /* automatically registered to PetscFinalize() when first context is instantiated, do not
    641:    call */
    642: static PetscErrorCode PetscDeviceContextDestroyGlobalContext_Private(void)
    643: {

    647:   PetscDeviceContextSynchronize(globalContext);
    648:   PetscDeviceContextDestroy(&globalContext);
    649:   /* reset everything to defaults */
    650:   defaultStreamType  = PETSC_STREAM_DEFAULT_BLOCKING;
    651:   globalContextSetup = PETSC_FALSE;
    652:   return(0);
    653: }

    655: /* creates and initializes the root context in PetscInitialize() but does not call
    656:    SetUp() as the user may wish to change types after PetscInitialize() */
    657: PetscErrorCode PetscDeviceContextInitializeRootContext_Internal(MPI_Comm comm, const char prefix[])
    658: {

    662:   PetscInfo1(NULL,"Initializing root PetscDeviceContext with PetscDeviceKind %s\n",PetscDeviceKinds[PETSC_DEVICE_DEFAULT]);
    663:   PetscDeviceContextCreate(&globalContext);
    664:   if (PetscUnlikelyDebug(globalContext->id != 0)) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_PLIB,"The root current PetscDeviceContext should have id = 0, however it has id = %D",globalContext->id);
    665:   PetscDeviceContextSetDevice(globalContext,PetscDeviceDefault_Internal());
    666:   PetscDeviceContextSetStreamType(globalContext,defaultStreamType);
    667:   PetscDeviceContextSetFromOptions(comm,prefix,globalContext);
    668:   PetscRegisterFinalize(PetscDeviceContextDestroyGlobalContext_Private);
    669:   return(0);
    670: }

    672: /*@C
    673:   PetscDeviceContextGetCurrentContext - Get the current active PetscDeviceContext

    675:   Not Collective, Asynchronous

    677:   Output Parameter:
    678: . dctx - The PetscDeviceContext

    680:   Notes:
    681:   The user generally should not destroy contexts retrieved with this routine unless they themselves have created
    682:   them. There exists no protection against destroying the root context.

    684:   Developer Notes:
    685:   This routine creates the "root" context the first time it is called, registering its
    686:   destructor to PetscFinalize(). The root context is synchronized before being destroyed.

    688:   Level: beginner

    690: .seealso: PetscDeviceContextSetCurrentContext(), PetscDeviceContextFork(),
    691: PetscDeviceContextJoin(), PetscDeviceContextCreate()
    692: @*/
    693: PetscErrorCode PetscDeviceContextGetCurrentContext(PetscDeviceContext *dctx)
    694: {
    697:   if (PetscUnlikely(!globalContextSetup)) {

    700:     /* if there is no available device backend, PetscDeviceInitializePackage() will fire a
    701:        PETSC_ERR_SUP_SYS error. */
    702:     PetscDeviceInitializePackage();
    703:     PetscDeviceContextSetUp(globalContext);
    704:     globalContextSetup = PETSC_TRUE;
    705:   }
    706:   *dctx = globalContext;
    707:   return(0);
    708: }

    710: /*@C
    711:   PetscDeviceContextSetCurrentContext - Set the current active PetscDeviceContext

    713:   Not Collective, Asynchronous

    715:   Input Parameter:
    716: . dctx - The PetscDeviceContext

    718:   Notes:
    719:   The old context is not stored in any way by this routine; if one is overriding a context that they themselves do not
    720:   control, one should take care to temporarily store it by calling PetscDeviceContextGetCurrentContext() before calling
    721:   this routine.

    723:   Level: beginner

    725: .seealso: PetscDeviceContextGetCurrentContext(), PetscDeviceContextFork(),
    726: PetscDeviceContextJoin(), PetscDeviceContextCreate()
    727: @*/
    728: PetscErrorCode PetscDeviceContextSetCurrentContext(PetscDeviceContext dctx)
    729: {

    734:   globalContext = dctx;
    735:   PetscInfo1(NULL,"Set global device context id %D\n",dctx->id);
    736:   return(0);
    737: }

    739: /*@C
    740:   PetscDeviceContextSetFromOptions - Configure a PetscDeviceContext from the options database

    742:   Collective on comm, Asynchronous

    744:   Input Parameters:
    745: + comm   - MPI communicator on which to query the options database
    746: . prefix - prefix to prepend to all options database queries, NULL if not needed
    747: - dctx   - The PetscDeviceContext to configure

    749:   Output Parameter:
    750: . dctx - The PetscDeviceContext

    752:   Options Database:
    753: + -device_context_device_kind - the kind of PetscDevice to attach by default - PetscDeviceKind
    754: - -device_context_stream_type - type of stream to create inside the PetscDeviceContext -
    755:   PetscDeviceContextSetStreamType()

    757:   Level: beginner

    759: .seealso: PetscDeviceContextSetStreamType(), PetscDeviceContextSetDevice()
    760: @*/
    761: PetscErrorCode PetscDeviceContextSetFromOptions(MPI_Comm comm, const char prefix[], PetscDeviceContext dctx)
    762: {
    763:   PetscBool      flag;
    764:   PetscInt       stype,dkind;

    770:   PetscOptionsBegin(comm,prefix,"PetscDeviceContext Options","Sys");
    771:   PetscOptionsEList("-device_context_device_kind","Underlying PetscDevice","PetscDeviceContextSetDevice",PetscDeviceKinds+1,PETSC_DEVICE_MAX-1,dctx->device ? PetscDeviceKinds[dctx->device->kind] : PetscDeviceKinds[PETSC_DEVICE_DEFAULT],&dkind,&flag);
    772:   if (flag) {
    773:     PetscDeviceContextSetDevice(dctx,PetscDeviceDefaultKind_Internal(static_cast<PetscDeviceKind>(dkind+1)));
    774:   }
    775:   PetscOptionsEList("-device_context_stream_type","PetscDeviceContext PetscStreamType","PetscDeviceContextSetStreamType",PetscStreamTypes,3,PetscStreamTypes[dctx->streamType],&stype,&flag);
    776:   if (flag) {
    777:     PetscDeviceContextSetStreamType(dctx,static_cast<PetscStreamType>(stype));
    778:   }
    779:   PetscOptionsEnd();
    780:   return(0);
    781: }