Date: 2022-03-04T17:37:08+00:00

Actual source code: init.c

      1: /*

      3:    This file defines part of the initialization of PETSc

      5:   This file uses regular malloc and free because it cannot be known
      6:   what malloc is being used until it has already processed the input.
      7: */

      9: #include <petscsys.h>
     10: #include <petsc/private/petscimpl.h>
     11: #include <petscviewer.h>
     12: #if defined(PETSC_USE_LOG)
     13: PETSC_INTERN PetscErrorCode PetscLogInitialize(void);
     14: #endif

     16: #if defined(PETSC_HAVE_SYS_SYSINFO_H)
     17: #include <sys/sysinfo.h>
     18: #endif
     19: #if defined(PETSC_HAVE_UNISTD_H)
     20: #include <unistd.h>
     21: #endif

     23: #if defined(PETSC_HAVE_CUDA) || defined(PETSC_HAVE_HIP)
     24: #  include <petscdevice.h>
     25: #endif

     27: #if defined(PETSC_HAVE_DEVICE)
     28:   #if defined(PETSC_HAVE_OMPI_MAJOR_VERSION)
     29:     #include "mpi-ext.h" /* Needed for OpenMPI CUDA-aware check */
     30:   #endif
     31: #endif

     33: #if defined(PETSC_HAVE_VIENNACL)
     34: PETSC_EXTERN PetscErrorCode PetscViennaCLInit();
     35: #endif

     37: /* ------------------------Nasty global variables -------------------------------*/
     38: /*
     39:      Indicates if PETSc started up MPI, or it was
     40:    already started before PETSc was initialized.
     41: */
     42: PetscBool   PetscBeganMPI                 = PETSC_FALSE;
     43: PetscBool   PetscErrorHandlingInitialized = PETSC_FALSE;
     44: PetscBool   PetscInitializeCalled         = PETSC_FALSE;
     45: PetscBool   PetscFinalizeCalled           = PETSC_FALSE;

     47: PetscMPIInt PetscGlobalRank               = -1;
     48: PetscMPIInt PetscGlobalSize               = -1;

     50: #if defined(PETSC_HAVE_KOKKOS)
     51: PetscBool   PetscBeganKokkos              = PETSC_FALSE;
     52: #endif

     54: #if defined(PETSC_HAVE_NVSHMEM)
     55: PetscBool   PetscBeganNvshmem             = PETSC_FALSE;
     56: PetscBool   PetscNvshmemInitialized       = PETSC_FALSE;
     57: #endif

     59: PetscBool   use_gpu_aware_mpi             = PETSC_TRUE;
     60: PetscBool   PetscCreatedGpuObjects        = PETSC_FALSE;

     62: #if defined(PETSC_HAVE_COMPLEX)
     63: #if defined(PETSC_COMPLEX_INSTANTIATE)
     64: template <> class std::complex<double>; /* instantiate complex template class */
     65: #endif

     67: /*MC
     68:    PETSC_i - the imaginary number i

     70:    Synopsis:
     71: #include <petscsys.h>
     72:    PetscComplex PETSC_i;

     74:    Level: beginner

     76:    Note:
     77:    Complex numbers are automatically available if PETSc located a working complex implementation

     79: .seealso: PetscRealPart(), PetscImaginaryPart(), PetscRealPartComplex(), PetscImaginaryPartComplex()
     80: M*/
     81: PetscComplex PETSC_i;
     82: MPI_Datatype MPIU___COMPLEX128 = 0;
     83: #endif /* PETSC_HAVE_COMPLEX */
     84: #if defined(PETSC_USE_REAL___FLOAT128)
     85: MPI_Datatype MPIU___FLOAT128 = 0;
     86: #elif defined(PETSC_USE_REAL___FP16)
     87: MPI_Datatype MPIU___FP16 = 0;
     88: #endif
     89: MPI_Datatype MPIU_2SCALAR = 0;
     90: MPI_Datatype MPIU_REAL_INT = 0;
     91: MPI_Datatype MPIU_SCALAR_INT = 0;
     92: #if defined(PETSC_USE_64BIT_INDICES)
     93: MPI_Datatype MPIU_2INT = 0;
     94: #endif
     95: MPI_Datatype MPI_4INT = 0;
     96: MPI_Datatype MPIU_4INT = 0;
     97: MPI_Datatype MPIU_BOOL;
     98: MPI_Datatype MPIU_ENUM;
     99: MPI_Datatype MPIU_FORTRANADDR;
    100: MPI_Datatype MPIU_SIZE_T;

    102: /*
    103:        Function that is called to display all error messages
    104: */
    105: PetscErrorCode (*PetscErrorPrintf)(const char [],...)          = PetscErrorPrintfDefault;
    106: PetscErrorCode (*PetscHelpPrintf)(MPI_Comm,const char [],...)  = PetscHelpPrintfDefault;
    107: PetscErrorCode (*PetscVFPrintf)(FILE*,const char[],va_list)    = PetscVFPrintfDefault;
    108: /*
    109:   This is needed to turn on/off GPU synchronization
    110: */
    111: PetscBool PetscViennaCLSynchronize = PETSC_FALSE;

    113: /* ------------------------------------------------------------------------------*/
    114: /*
    115:    Optional file where all PETSc output from various prints is saved
    116: */
    117: PETSC_INTERN FILE *petsc_history;
    118: FILE *petsc_history = NULL;

    120: PetscErrorCode  PetscOpenHistoryFile(const char filename[],FILE **fd)
    121: {
    123:   PetscMPIInt    rank,size;
    124:   char           pfile[PETSC_MAX_PATH_LEN],pname[PETSC_MAX_PATH_LEN],fname[PETSC_MAX_PATH_LEN],date[64];
    125:   char           version[256];

    128:   MPI_Comm_rank(PETSC_COMM_WORLD,&rank);
    129:   if (rank == 0) {
    130:     char        arch[10];
    131:     int         err;

    133:     PetscGetArchType(arch,10);
    134:     PetscGetDate(date,64);
    135:     PetscGetVersion(version,256);
    136:     MPI_Comm_size(PETSC_COMM_WORLD,&size);
    137:     if (filename) {
    138:       PetscFixFilename(filename,fname);
    139:     } else {
    140:       PetscGetHomeDirectory(pfile,sizeof(pfile));
    141:       PetscStrlcat(pfile,"/.petschistory",sizeof(pfile));
    142:       PetscFixFilename(pfile,fname);
    143:     }

    145:     *fd = fopen(fname,"a");
    146:     if (!fd) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_FILE_OPEN,"Cannot open file: %s",fname);

    148:     PetscFPrintf(PETSC_COMM_SELF,*fd,"----------------------------------------\n");
    149:     PetscFPrintf(PETSC_COMM_SELF,*fd,"%s %s\n",version,date);
    150:     PetscGetProgramName(pname,sizeof(pname));
    151:     PetscFPrintf(PETSC_COMM_SELF,*fd,"%s on a %s, %d proc. with options:\n",pname,arch,size);
    152:     PetscFPrintf(PETSC_COMM_SELF,*fd,"----------------------------------------\n");

    154:     err = fflush(*fd);
    155:     if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fflush() failed on file");
    156:   }
    157:   return(0);
    158: }

    160: PETSC_INTERN PetscErrorCode PetscCloseHistoryFile(FILE **fd)
    161: {
    163:   PetscMPIInt    rank;
    164:   char           date[64];
    165:   int            err;

    168:   MPI_Comm_rank(PETSC_COMM_WORLD,&rank);
    169:   if (rank == 0) {
    170:     PetscGetDate(date,64);
    171:     PetscFPrintf(PETSC_COMM_SELF,*fd,"----------------------------------------\n");
    172:     PetscFPrintf(PETSC_COMM_SELF,*fd,"Finished at %s\n",date);
    173:     PetscFPrintf(PETSC_COMM_SELF,*fd,"----------------------------------------\n");
    174:     err  = fflush(*fd);
    175:     if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fflush() failed on file");
    176:     err = fclose(*fd);
    177:     if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fclose() failed on file");
    178:   }
    179:   return(0);
    180: }

    182: /* ------------------------------------------------------------------------------*/

    184: /*
    185:    This is ugly and probably belongs somewhere else, but I want to
    186:   be able to put a true MPI abort error handler with command line args.

    188:     This is so MPI errors in the debugger will leave all the stack
    189:   frames. The default MP_Abort() cleans up and exits thus providing no useful information
    190:   in the debugger hence we call abort() instead of MPI_Abort().
    191: */

    193: void Petsc_MPI_AbortOnError(MPI_Comm *comm,PetscMPIInt *flag,...)
    194: {
    196:   (*PetscErrorPrintf)("MPI error %d\n",*flag);
    197:   abort();
    198: }

    200: void Petsc_MPI_DebuggerOnError(MPI_Comm *comm,PetscMPIInt *flag,...)
    201: {

    205:   (*PetscErrorPrintf)("MPI error %d\n",*flag);
    206:   PetscAttachDebugger();
    207:   if (ierr) PETSCABORT(*comm,*flag); /* hopeless so get out */
    208: }

    210: /*@C
    211:    PetscEnd - Calls PetscFinalize() and then ends the program. This is useful if one
    212:      wishes a clean exit somewhere deep in the program.

    214:    Collective on PETSC_COMM_WORLD

    216:    Options Database Keys are the same as for PetscFinalize()

    218:    Level: advanced

    220:    Note:
    221:    See PetscInitialize() for more general runtime options.

    223: .seealso: PetscInitialize(), PetscOptionsView(), PetscMallocDump(), PetscMPIDump(), PetscFinalize()
    224: @*/
    225: PetscErrorCode  PetscEnd(void)
    226: {
    228:   PetscFinalize();
    229:   exit(0);
    230:   return 0;
    231: }

    233: PetscBool PetscOptionsPublish = PETSC_FALSE;
    234: PETSC_INTERN PetscErrorCode PetscSetUseHBWMalloc_Private(void);
    235: PETSC_INTERN PetscBool      petscsetmallocvisited;
    236: static       char           emacsmachinename[256];

    238: PetscErrorCode (*PetscExternalVersionFunction)(MPI_Comm) = NULL;
    239: PetscErrorCode (*PetscExternalHelpFunction)(MPI_Comm)    = NULL;

    241: /*@C
    242:    PetscSetHelpVersionFunctions - Sets functions that print help and version information
    243:    before the PETSc help and version information is printed. Must call BEFORE PetscInitialize().
    244:    This routine enables a "higher-level" package that uses PETSc to print its messages first.

    246:    Input Parameters:
    247: +  help - the help function (may be NULL)
    248: -  version - the version function (may be NULL)

    250:    Level: developer

    252: @*/
    253: PetscErrorCode  PetscSetHelpVersionFunctions(PetscErrorCode (*help)(MPI_Comm),PetscErrorCode (*version)(MPI_Comm))
    254: {
    256:   PetscExternalHelpFunction    = help;
    257:   PetscExternalVersionFunction = version;
    258:   return(0);
    259: }

    261: #if defined(PETSC_USE_LOG)
    262: PETSC_INTERN PetscBool   PetscObjectsLog;
    263: #endif

    265: /* CUPM stands for 'CUDA Programming Model', which is implemented in either CUDA or HIP.
    266:    Use the following macros to define CUDA/HIP initialization related vars/routines.
    267:  */
    268: #if defined(PETSC_HAVE_CUDA)
    269:   typedef cudaError_t                             cupmError_t;
    270:   typedef struct cudaDeviceProp                   cupmDeviceProp;
    271:   typedef cudaStream_t                            cupmStream_t;
    272:   typedef cudaEvent_t                             cupmEvent_t;
    273:   #define cupmGetDeviceCount(x)                   cudaGetDeviceCount(x)
    274:   #define cupmGetDevice(x)                        cudaGetDevice(x)
    275:   #define cupmSetDevice(x)                        cudaSetDevice(x)
    276:   #define cupmSetDeviceFlags(x)                   cudaSetDeviceFlags(x)
    277:   #define cupmGetDeviceProperties(x,y)            cudaGetDeviceProperties(x,y)
    278:   #define cupmStreamCreate(x)                     cudaStreamCreate(x)
    279:   #define cupmGetLastError()                      cudaGetLastError()
    280:   #define cupmDeviceMapHost                       cudaDeviceMapHost
    281:   #define cupmSuccess                             cudaSuccess
    282:   #define cupmErrorMemoryAllocation               cudaErrorMemoryAllocation
    283:   #define cupmErrorLaunchOutOfResources           cudaErrorLaunchOutOfResources
    284:   #define cupmErrorSetOnActiveProcess             cudaErrorSetOnActiveProcess
    285:   #define CHKERRCUPM(x)                           CHKERRCUDA(x)
    286:   #define PetscCUPMBLASInitializeHandle()         PetscCUBLASInitializeHandle()
    287:   #define PetscCUPMSOLVERDnInitializeHandle()     PetscCUSOLVERDnInitializeHandle()
    288:   #define PetscCUPMInitialize                     PetscCUDAInitialize
    289:   #define PetscCUPMInitialized                    PetscCUDAInitialized
    290:   #define PetscCUPMInitializeCheck                PetscCUDAInitializeCheck
    291:   #define PetscCUPMInitializeAndView              PetscCUDAInitializeAndView
    292:   #define PetscCUPMSynchronize                    PetscCUDASynchronize
    293:   #define PetscNotUseCUPM                         PetscNotUseCUDA
    294:   #define cupmOptionsStr                          "CUDA options"
    295:   #define cupmSetDeviceStr                        "-cuda_device"
    296:   #define cupmViewStr                             "-cuda_view"
    297:   #define cupmSynchronizeStr                      "-cuda_synchronize"
    298:   #define PetscCUPMInitializeStr                  "PetscCUDAInitialize"
    299:   #define PetscOptionsCheckCUPM                   PetscOptionsCheckCUDA
    300:   #define PetscMPICUPMAwarenessCheck              PetscMPICUDAAwarenessCheck
    301:   #define PetscDefaultCupmStream                  PetscDefaultCudaStream
    302:   #define cupmEventCreate(x)                      cudaEventCreate(x)
    303:   #include "cupminit.inc"
    304: #endif

    306: #if defined(PETSC_HAVE_HIP)
    307:   typedef hipError_t                              cupmError_t;
    308:   typedef hipDeviceProp_t                         cupmDeviceProp;
    309:   typedef hipStream_t                             cupmStream_t;
    310:   typedef hipEvent_t                              cupmEvent_t;
    311:   #define cupmGetDeviceCount(x)                   hipGetDeviceCount(x)
    312:   #define cupmGetDevice(x)                        hipGetDevice(x)
    313:   #define cupmSetDevice(x)                        hipSetDevice(x)
    314:   #define cupmSetDeviceFlags(x)                   hipSetDeviceFlags(x)
    315:   #define cupmGetDeviceProperties(x,y)            hipGetDeviceProperties(x,y)
    316:   #define cupmStreamCreate(x)                     hipStreamCreate(x)
    317:   #define cupmEventCreate(x)                      hipEventCreate(x);
    318:   #define cupmGetLastError()                      hipGetLastError()
    319:   #define cupmDeviceMapHost                       hipDeviceMapHost
    320:   #define cupmSuccess                             hipSuccess
    321:   #define cupmErrorMemoryAllocation               hipErrorMemoryAllocation
    322:   #define cupmErrorLaunchOutOfResources           hipErrorLaunchOutOfResources
    323:   #define cupmErrorSetOnActiveProcess             hipErrorSetOnActiveProcess
    324:   #define CHKERRCUPM(x)                           CHKERRQ((x)==hipSuccess? 0:PETSC_ERR_LIB)
    325:   #define PetscCUPMBLASInitializeHandle()         0
    326:   #define PetscCUPMSOLVERDnInitializeHandle()     0
    327:   #define PetscCUPMInitialize                     PetscHIPInitialize
    328:   #define PetscCUPMInitialized                    PetscHIPInitialized
    329:   #define PetscCUPMInitializeCheck                PetscHIPInitializeCheck
    330:   #define PetscCUPMInitializeAndView              PetscHIPInitializeAndView
    331:   #define PetscCUPMSynchronize                    PetscHIPSynchronize
    332:   #define PetscNotUseCUPM                         PetscNotUseHIP
    333:   #define cupmOptionsStr                          "HIP options"
    334:   #define cupmSetDeviceStr                        "-hip_device"
    335:   #define cupmViewStr                             "-hip_view"
    336:   #define cupmSynchronizeStr                      "-hip_synchronize"
    337:   #define PetscCUPMInitializeStr                  "PetscHIPInitialize"
    338:   #define PetscOptionsCheckCUPM                   PetscOptionsCheckHIP
    339:   #define PetscMPICUPMAwarenessCheck              PetscMPIHIPAwarenessCheck
    340:   #define PetscDefaultCupmStream                  PetscDefaultHipStream
    341:   #include "cupminit.inc"
    342: #endif

    344: PETSC_INTERN PetscErrorCode  PetscOptionsCheckInitial_Private(const char help[])
    345: {
    346:   char              string[64];
    347:   MPI_Comm          comm = PETSC_COMM_WORLD;
    348:   PetscBool         flg1 = PETSC_FALSE,flg2 = PETSC_FALSE,flg3 = PETSC_FALSE,flag,hasHelp,logView;
    349:   PetscErrorCode    ierr;
    350:   PetscReal         si;
    351:   PetscInt          intensity;
    352:   int               i;
    353:   PetscMPIInt       rank;
    354:   char              version[256];
    355: #if defined(PETSC_USE_LOG)
    356:   char              mname[PETSC_MAX_PATH_LEN];
    357:   PetscViewerFormat format;
    358:   PetscBool         flg4 = PETSC_FALSE;
    359: #endif

    362:   MPI_Comm_rank(comm,&rank);

    364:   /*
    365:      Setup building of stack frames for all function calls
    366:   */
    367:   if (PetscDefined(USE_DEBUG) && !PetscDefined(HAVE_THREADSAFETY)) {
    368:     PetscOptionsGetBool(NULL,NULL,"-checkstack",&flg1,NULL);
    369:     PetscStackSetCheck(flg1);
    370:   }

    372: #if !defined(PETSC_HAVE_THREADSAFETY)
    373:   if (!(PETSC_RUNNING_ON_VALGRIND)) {
    374:     /*
    375:       Setup the memory management; support for tracing malloc() usage
    376:     */
    377:     PetscBool mdebug = PETSC_FALSE, eachcall = PETSC_FALSE, initializenan = PETSC_FALSE, mlog = PETSC_FALSE;

    379:     if (PetscDefined(USE_DEBUG)) {
    380:       mdebug        = PETSC_TRUE;
    381:       initializenan = PETSC_TRUE;
    382:       PetscOptionsHasName(NULL,NULL,"-malloc_test",&flg1);
    383:     } else {
    384:       /* don't warn about unused option */
    385:       PetscOptionsHasName(NULL,NULL,"-malloc_test",&flg1);
    386:       flg1 = PETSC_FALSE;
    387:     }
    388:     PetscOptionsGetBool(NULL,NULL,"-malloc_debug",&flg2,&flg3);
    389:     if (flg1 || flg2) {
    390:       mdebug        = PETSC_TRUE;
    391:       eachcall      = PETSC_TRUE;
    392:       initializenan = PETSC_TRUE;
    393:     } else if (flg3 && !flg2) {
    394:       mdebug        = PETSC_FALSE;
    395:       eachcall      = PETSC_FALSE;
    396:       initializenan = PETSC_FALSE;
    397:     }

    399:     PetscOptionsGetBool(NULL,NULL,"-malloc_requested_size",&flg1,&flg2);
    400:     if (flg2) {PetscMallocLogRequestedSizeSet(flg1);}

    402:     PetscOptionsHasName(NULL,NULL,"-malloc_view",&mlog);
    403:     if (mlog) {
    404:       mdebug = PETSC_TRUE;
    405:     }
    406:     /* the next line is deprecated */
    407:     PetscOptionsGetBool(NULL,NULL,"-malloc",&mdebug,NULL);
    408:     PetscOptionsGetBool(NULL,NULL,"-malloc_dump",&mdebug,NULL);
    409:     PetscOptionsGetBool(NULL,NULL,"-log_view_memory",&mdebug,NULL);
    410:     if (mdebug) {
    411:       PetscMallocSetDebug(eachcall,initializenan);
    412:     }
    413:     if (mlog) {
    414:       PetscReal logthreshold = 0;
    415:       PetscOptionsGetReal(NULL,NULL,"-malloc_view_threshold",&logthreshold,NULL);
    416:       PetscMallocViewSet(logthreshold);
    417:     }
    418: #if defined(PETSC_USE_LOG)
    419:     PetscOptionsGetBool(NULL,NULL,"-log_view_memory",&PetscLogMemory,NULL);
    420: #endif
    421:   }

    423:   PetscOptionsGetBool(NULL,NULL,"-malloc_coalesce",&flg1,&flg2);
    424:   if (flg2) {PetscMallocSetCoalesce(flg1);}
    425:   flg1 = PETSC_FALSE;
    426:   PetscOptionsGetBool(NULL,NULL,"-malloc_hbw",&flg1,NULL);
    427:   /* ignore this option if malloc is already set */
    428:   if (flg1 && !petscsetmallocvisited) {PetscSetUseHBWMalloc_Private();}

    430:   flg1 = PETSC_FALSE;
    431:   PetscOptionsGetBool(NULL,NULL,"-malloc_info",&flg1,NULL);
    432:   if (!flg1) {
    433:     flg1 = PETSC_FALSE;
    434:     PetscOptionsGetBool(NULL,NULL,"-memory_view",&flg1,NULL);
    435:   }
    436:   if (flg1) {
    437:     PetscMemorySetGetMaximumUsage();
    438:   }
    439: #endif

    441: #if defined(PETSC_USE_LOG)
    442:   PetscOptionsHasName(NULL,NULL,"-objects_dump",&PetscObjectsLog);
    443: #endif

    445:   /*
    446:       Set the display variable for graphics
    447:   */
    448:   PetscSetDisplay();

    450:   /*
    451:      Print main application help message
    452:   */
    453:   PetscOptionsHasHelp(NULL,&hasHelp);
    454:   if (help && hasHelp) {
    455:     PetscPrintf(comm,help);
    456:     PetscPrintf(comm,"----------------------------------------\n");
    457:   }

    459:   /*
    460:       Print the PETSc version information
    461:   */
    462:   PetscOptionsHasName(NULL,NULL,"-version",&flg1);
    463:   if (flg1 || hasHelp) {
    464:     /*
    465:        Print "higher-level" package version message
    466:     */
    467:     if (PetscExternalVersionFunction) {
    468:       (*PetscExternalVersionFunction)(comm);
    469:     }

    471:     PetscGetVersion(version,256);
    472:     (*PetscHelpPrintf)(comm,"%s\n",version);
    473:     (*PetscHelpPrintf)(comm,"%s",PETSC_AUTHOR_INFO);
    474:     (*PetscHelpPrintf)(comm,"See docs/changes/index.html for recent updates.\n");
    475:     (*PetscHelpPrintf)(comm,"See docs/faq.html for problems.\n");
    476:     (*PetscHelpPrintf)(comm,"See docs/manualpages/index.html for help. \n");
    477:     (*PetscHelpPrintf)(comm,"Libraries linked from %s\n",PETSC_LIB_DIR);
    478:     (*PetscHelpPrintf)(comm,"----------------------------------------\n");
    479:   }

    481:   /*
    482:        Print "higher-level" package help message
    483:   */
    484:   if (hasHelp) {
    485:     PetscBool hasHelpIntro;

    487:     if (PetscExternalHelpFunction) {
    488:       (*PetscExternalHelpFunction)(comm);
    489:     }
    490:     PetscOptionsHasHelpIntro_Internal(NULL,&hasHelpIntro);
    491:     if (hasHelpIntro) {
    492:       PetscOptionsDestroyDefault();
    493:       PetscFreeMPIResources();
    494:       MPI_Finalize();
    495:       exit(0);
    496:     }
    497:   }

    499:   /*
    500:       Setup the error handling
    501:   */
    502:   flg1 = PETSC_FALSE;
    503:   PetscOptionsGetBool(NULL,NULL,"-on_error_abort",&flg1,NULL);
    504:   if (flg1) {
    505:     MPI_Comm_set_errhandler(comm,MPI_ERRORS_ARE_FATAL);
    506:     PetscPushErrorHandler(PetscAbortErrorHandler,NULL);
    507:   }
    508:   flg1 = PETSC_FALSE;
    509:   PetscOptionsGetBool(NULL,NULL,"-on_error_mpiabort",&flg1,NULL);
    510:   if (flg1) { PetscPushErrorHandler(PetscMPIAbortErrorHandler,NULL);}
    511:   flg1 = PETSC_FALSE;
    512:   PetscOptionsGetBool(NULL,NULL,"-mpi_return_on_error",&flg1,NULL);
    513:   if (flg1) {
    514:     MPI_Comm_set_errhandler(comm,MPI_ERRORS_RETURN);
    515:   }
    516:   flg1 = PETSC_FALSE;
    517:   PetscOptionsGetBool(NULL,NULL,"-no_signal_handler",&flg1,NULL);
    518:   if (!flg1) {PetscPushSignalHandler(PetscSignalHandlerDefault,(void*)0);}

    520:   /*
    521:       Setup debugger information
    522:   */
    523:   PetscSetDefaultDebugger();
    524:   PetscOptionsGetString(NULL,NULL,"-on_error_attach_debugger",string,sizeof(string),&flg1);
    525:   if (flg1) {
    526:     MPI_Errhandler err_handler;

    528:     PetscSetDebuggerFromString(string);
    529:     MPI_Comm_create_errhandler(Petsc_MPI_DebuggerOnError,&err_handler);
    530:     MPI_Comm_set_errhandler(comm,err_handler);
    531:     PetscPushErrorHandler(PetscAttachDebuggerErrorHandler,NULL);
    532:   }
    533:   PetscOptionsGetString(NULL,NULL,"-debug_terminal",string,sizeof(string),&flg1);
    534:   if (flg1) { PetscSetDebugTerminal(string); }
    535:   PetscOptionsGetString(NULL,NULL,"-start_in_debugger",string,sizeof(string),&flg1);
    536:   PetscOptionsGetString(NULL,NULL,"-stop_for_debugger",string,sizeof(string),&flg2);
    537:   if (flg1 || flg2) {
    538:     PetscMPIInt    size;
    539:     PetscInt       lsize,*ranks;
    540:     MPI_Errhandler err_handler;
    541:     /*
    542:        we have to make sure that all processors have opened
    543:        connections to all other processors, otherwise once the
    544:        debugger has stated it is likely to receive a SIGUSR1
    545:        and kill the program.
    546:     */
    547:     MPI_Comm_size(comm,&size);
    548:     if (size > 2) {
    549:       PetscMPIInt dummy = 0;
    550:       MPI_Status  status;
    551:       for (i=0; i<size; i++) {
    552:         if (rank != i) {
    553:           MPI_Send(&dummy,1,MPI_INT,i,109,comm);
    554:         }
    555:       }
    556:       for (i=0; i<size; i++) {
    557:         if (rank != i) {
    558:           MPI_Recv(&dummy,1,MPI_INT,i,109,comm,&status);
    559:         }
    560:       }
    561:     }
    562:     /* check if this processor node should be in debugger */
    563:     PetscMalloc1(size,&ranks);
    564:     lsize = size;
    565:     /* Deprecated in 3.14 */
    566:     PetscOptionsGetIntArray(NULL,NULL,"-debugger_nodes",ranks,&lsize,&flag);
    567:     if (flag) {
    568:       const char * const quietopt="-options_suppress_deprecated_warnings";
    569:       char               msg[4096];
    570:       PetscBool          quiet = PETSC_FALSE;

    572:       PetscOptionsGetBool(NULL,NULL,quietopt,&quiet,NULL);
    573:       if (!quiet) {
    574:         PetscStrcpy(msg,"** PETSc DEPRECATION WARNING ** : the option ");
    575:         PetscStrcat(msg,"-debugger_nodes");
    576:         PetscStrcat(msg," is deprecated as of version ");
    577:         PetscStrcat(msg,"3.14");
    578:         PetscStrcat(msg," and will be removed in a future release.");
    579:         PetscStrcat(msg," Please use the option ");
    580:         PetscStrcat(msg,"-debugger_ranks");
    581:         PetscStrcat(msg," instead.");
    582:         PetscStrcat(msg," (Silence this warning with ");
    583:         PetscStrcat(msg,quietopt);
    584:         PetscStrcat(msg,")\n");
    585:         PetscPrintf(comm,msg);
    586:       }
    587:     } else {
    588:       lsize = size;
    589:       PetscOptionsGetIntArray(NULL,NULL,"-debugger_ranks",ranks,&lsize,&flag);
    590:     }
    591:     if (flag) {
    592:       for (i=0; i<lsize; i++) {
    593:         if (ranks[i] == rank) { flag = PETSC_FALSE; break; }
    594:       }
    595:     }
    596:     if (!flag) {
    597:       PetscSetDebuggerFromString(string);
    598:       PetscPushErrorHandler(PetscAbortErrorHandler,NULL);
    599:       if (flg1) {
    600:         PetscAttachDebugger();
    601:       } else {
    602:         PetscStopForDebugger();
    603:       }
    604:       MPI_Comm_create_errhandler(Petsc_MPI_AbortOnError,&err_handler);
    605:       MPI_Comm_set_errhandler(comm,err_handler);
    606:     } else {
    607:       PetscWaitOnError();
    608:     }
    609:     PetscFree(ranks);
    610:   }

    612:   PetscOptionsGetString(NULL,NULL,"-on_error_emacs",emacsmachinename,sizeof(emacsmachinename),&flg1);
    613:   if (flg1 && rank == 0) {PetscPushErrorHandler(PetscEmacsClientErrorHandler,emacsmachinename);}

    615:   /*
    616:         Setup profiling and logging
    617:   */
    618: #if defined(PETSC_USE_INFO)
    619:   {
    620:     PetscInfoSetFromOptions(NULL);
    621:   }
    622: #endif
    623:   PetscDetermineInitialFPTrap();
    624:   flg1 = PETSC_FALSE;
    625:   PetscOptionsGetBool(NULL,NULL,"-fp_trap",&flg1,&flag);
    626:   if (flag) {PetscSetFPTrap((PetscFPTrap)flg1);}
    627:   PetscOptionsGetInt(NULL,NULL,"-check_pointer_intensity",&intensity,&flag);
    629: #if defined(PETSC_USE_LOG)
    630:   mname[0] = 0;
    631:   PetscOptionsGetString(NULL,NULL,"-history",mname,sizeof(mname),&flg1);
    632:   if (flg1) {
    633:     if (mname[0]) {
    634:       PetscOpenHistoryFile(mname,&petsc_history);
    635:     } else {
    636:       PetscOpenHistoryFile(NULL,&petsc_history);
    637:     }
    638:   }

    640:   PetscOptionsGetBool(NULL,NULL,"-log_sync",&PetscLogSyncOn,NULL);

    642: #if defined(PETSC_HAVE_MPE)
    643:   flg1 = PETSC_FALSE;
    644:   PetscOptionsHasName(NULL,NULL,"-log_mpe",&flg1);
    645:   if (flg1) {PetscLogMPEBegin();}
    646: #endif
    647:   flg1 = PETSC_FALSE;
    648:   flg3 = PETSC_FALSE;
    649:   PetscOptionsGetBool(NULL,NULL,"-log_all",&flg1,NULL);
    650:   PetscOptionsHasName(NULL,NULL,"-log_summary",&flg3);
    651:   if (flg1)                      { PetscLogAllBegin(); }
    652:   else if (flg3)                 { PetscLogDefaultBegin();}

    654:   PetscOptionsGetString(NULL,NULL,"-log_trace",mname,sizeof(mname),&flg1);
    655:   if (flg1) {
    656:     char name[PETSC_MAX_PATH_LEN],fname[PETSC_MAX_PATH_LEN];
    657:     FILE *file;
    658:     if (mname[0]) {
    659:       PetscSNPrintf(name,PETSC_MAX_PATH_LEN,"%s.%d",mname,rank);
    660:       PetscFixFilename(name,fname);
    661:       file = fopen(fname,"w");
    662:       if (!file) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_FILE_OPEN,"Unable to open trace file: %s",fname);
    663:     } else file = PETSC_STDOUT;
    664:     PetscLogTraceBegin(file);
    665:   }

    667:   PetscOptionsGetViewer(comm,NULL,NULL,"-log_view",NULL,&format,&flg4);
    668:   if (flg4) {
    669:     if (format == PETSC_VIEWER_ASCII_XML || format == PETSC_VIEWER_ASCII_FLAMEGRAPH) {
    670:       PetscLogNestedBegin();
    671:     } else {
    672:       PetscLogDefaultBegin();
    673:     }
    674:   }
    675:   if (flg4 && (format == PETSC_VIEWER_ASCII_XML || format == PETSC_VIEWER_ASCII_FLAMEGRAPH)) {
    676:     PetscReal threshold = PetscRealConstant(0.01);
    677:     PetscOptionsGetReal(NULL,NULL,"-log_threshold",&threshold,&flg1);
    678:     if (flg1) {PetscLogSetThreshold((PetscLogDouble)threshold,NULL);}
    679:   }
    680: #endif

    682:   PetscOptionsGetBool(NULL,NULL,"-saws_options",&PetscOptionsPublish,NULL);
    683:   PetscOptionsGetBool(NULL,NULL,"-use_gpu_aware_mpi",&use_gpu_aware_mpi,NULL);
    684:   /*
    685:     If collecting logging information, by default, wait for device to complete its operations
    686:     before returning to the CPU in order to get accurate timings of each event
    687:   */
    688:   PetscOptionsHasName(NULL,NULL,"-log_summary",&logView);
    689:   if (!logView) {PetscOptionsHasName(NULL,NULL,"-log_view",&logView);}

    691: #if defined(PETSC_HAVE_CUDA)
    692:   PetscOptionsCheckCUDA(logView);
    693: #endif

    695: #if defined(PETSC_HAVE_HIP)
    696:   PetscOptionsCheckHIP(logView);
    697: #endif

    699:   /*
    700:        Print basic help message
    701:   */
    702:   if (hasHelp) {
    703:     (*PetscHelpPrintf)(comm,"Options for all PETSc programs:\n");
    704:     (*PetscHelpPrintf)(comm," -version: prints PETSc version\n");
    705:     (*PetscHelpPrintf)(comm," -help intro: prints example description and PETSc version, and exits\n");
    706:     (*PetscHelpPrintf)(comm," -help: prints example description, PETSc version, and available options for used routines\n");
    707:     (*PetscHelpPrintf)(comm," -on_error_abort: cause an abort when an error is detected. Useful \n ");
    708:     (*PetscHelpPrintf)(comm,"       only when run in the debugger\n");
    709:     (*PetscHelpPrintf)(comm," -on_error_attach_debugger [gdb,dbx,xxgdb,ups,noxterm]\n");
    710:     (*PetscHelpPrintf)(comm,"       start the debugger in new xterm\n");
    711:     (*PetscHelpPrintf)(comm,"       unless noxterm is given\n");
    712:     (*PetscHelpPrintf)(comm," -start_in_debugger [gdb,dbx,xxgdb,ups,noxterm]\n");
    713:     (*PetscHelpPrintf)(comm,"       start all processes in the debugger\n");
    714:     (*PetscHelpPrintf)(comm," -on_error_emacs <machinename>\n");
    715:     (*PetscHelpPrintf)(comm,"    emacs jumps to error file\n");
    716:     (*PetscHelpPrintf)(comm," -debugger_ranks [n1,n2,..] Ranks to start in debugger\n");
    717:     (*PetscHelpPrintf)(comm," -debugger_pause [m] : delay (in seconds) to attach debugger\n");
    718:     (*PetscHelpPrintf)(comm," -stop_for_debugger : prints message on how to attach debugger manually\n");
    719:     (*PetscHelpPrintf)(comm,"                      waits the delay for you to attach\n");
    720:     (*PetscHelpPrintf)(comm," -display display: Location where X window graphics and debuggers are displayed\n");
    721:     (*PetscHelpPrintf)(comm," -no_signal_handler: do not trap error signals\n");
    722:     (*PetscHelpPrintf)(comm," -mpi_return_on_error: MPI returns error code, rather than abort on internal error\n");
    723:     (*PetscHelpPrintf)(comm," -fp_trap: stop on floating point exceptions\n");
    724:     (*PetscHelpPrintf)(comm,"           note on IBM RS6000 this slows run greatly\n");
    725:     (*PetscHelpPrintf)(comm," -malloc_dump <optional filename>: dump list of unfreed memory at conclusion\n");
    726:     (*PetscHelpPrintf)(comm," -malloc: use PETSc error checking malloc (deprecated, use -malloc_debug)\n");
    727:     (*PetscHelpPrintf)(comm," -malloc no: don't use PETSc error checking malloc (deprecated, use -malloc_debug no)\n");
    728:     (*PetscHelpPrintf)(comm," -malloc_info: prints total memory usage\n");
    729:     (*PetscHelpPrintf)(comm," -malloc_view <optional filename>: keeps log of all memory allocations, displays in PetscFinalize()\n");
    730:     (*PetscHelpPrintf)(comm," -malloc_debug <true or false>: enables or disables extended checking for memory corruption\n");
    731:     (*PetscHelpPrintf)(comm," -options_view: dump list of options inputted\n");
    732:     (*PetscHelpPrintf)(comm," -options_left: dump list of unused options\n");
    733:     (*PetscHelpPrintf)(comm," -options_left no: don't dump list of unused options\n");
    734:     (*PetscHelpPrintf)(comm," -tmp tmpdir: alternative /tmp directory\n");
    735:     (*PetscHelpPrintf)(comm," -shared_tmp: tmp directory is shared by all processors\n");
    736:     (*PetscHelpPrintf)(comm," -not_shared_tmp: each processor has separate tmp directory\n");
    737:     (*PetscHelpPrintf)(comm," -memory_view: print memory usage at end of run\n");
    738: #if defined(PETSC_USE_LOG)
    739:     (*PetscHelpPrintf)(comm," -get_total_flops: total flops over all processors\n");
    740:     (*PetscHelpPrintf)(comm," -log_view [:filename:[format]]: logging objects and events\n");
    741:     (*PetscHelpPrintf)(comm," -log_trace [filename]: prints trace of all PETSc calls\n");
    742:     (*PetscHelpPrintf)(comm," -log_exclude <list,of,classnames>: exclude given classes from logging\n");
    743: #if defined(PETSC_HAVE_MPE)
    744:     (*PetscHelpPrintf)(comm," -log_mpe: Also create logfile viewable through Jumpshot\n");
    745: #endif
    746: #endif
    747: #if defined(PETSC_USE_INFO)
    748:     (*PetscHelpPrintf)(comm," -info [filename][:[~]<list,of,classnames>[:[~]self]]: print verbose information\n");
    749: #endif
    750:     (*PetscHelpPrintf)(comm," -options_file <file>: reads options from file\n");
    751:     (*PetscHelpPrintf)(comm," -options_monitor: monitor options to standard output, including that set previously e.g. in option files\n");
    752:     (*PetscHelpPrintf)(comm," -options_monitor_cancel: cancels all hardwired option monitors\n");
    753:     (*PetscHelpPrintf)(comm," -petsc_sleep n: sleeps n seconds before running program\n");
    754:   }

    756: #if defined(PETSC_HAVE_POPEN)
    757:   {
    758:   char machine[128];
    759:   PetscOptionsGetString(NULL,NULL,"-popen_machine",machine,sizeof(machine),&flg1);
    760:   if (flg1) {
    761:     PetscPOpenSetMachine(machine);
    762:   }
    763:   }
    764: #endif

    766:   PetscOptionsGetReal(NULL,NULL,"-petsc_sleep",&si,&flg1);
    767:   if (flg1) {
    768:     PetscSleep(si);
    769:   }

    771: #if defined(PETSC_HAVE_VIENNACL)
    772:   PetscOptionsHasName(NULL,NULL,"-log_summary",&flg3);
    773:   if (!flg3) {
    774:     PetscOptionsHasName(NULL,NULL,"-log_view",&flg3);
    775:   }
    776:   PetscOptionsGetBool(NULL,NULL,"-viennacl_synchronize",&flg3,NULL);
    777:   PetscViennaCLSynchronize = flg3;
    778:   PetscViennaCLInit();
    779: #endif

    781:   /*
    782:      Creates the logging data structures; this is enabled even if logging is not turned on
    783:      This is the last thing we do before returning to the user code to prevent having the
    784:      logging numbers contaminated by any startup time associated with MPI and the GPUs
    785:   */
    786: #if defined(PETSC_USE_LOG)
    787:   PetscLogInitialize();
    788: #endif

    790:   return(0);
    791: }