Date: 2022-03-31T04:46:06+00:00

Actual source code: mal.c

      1: /*
      2:     Code that allows a user to dictate what malloc() PETSc uses.
      3: */
      4: #include <petscsys.h>
      5: #include <stdarg.h>
      6: #if defined(PETSC_HAVE_MALLOC_H)
      7: #include <malloc.h>
      8: #endif
      9: #if defined(PETSC_HAVE_MEMKIND)
     10: #include <errno.h>
     11: #include <memkind.h>
     12: typedef enum {PETSC_MK_DEFAULT=0,PETSC_MK_HBW_PREFERRED=1} PetscMemkindType;
     13: PetscMemkindType currentmktype = PETSC_MK_HBW_PREFERRED;
     14: PetscMemkindType previousmktype = PETSC_MK_HBW_PREFERRED;
     15: #endif
     16: /*
     17:         We want to make sure that all mallocs of double or complex numbers are complex aligned.
     18:     1) on systems with memalign() we call that routine to get an aligned memory location
     19:     2) on systems without memalign() we
     20:        - allocate one sizeof(PetscScalar) extra space
     21:        - we shift the pointer up slightly if needed to get PetscScalar aligned
     22:        - if shifted we store at ptr[-1] the amount of shift (plus a classid)
     23: */
     24: #define SHIFT_CLASSID 456123

     26: PETSC_EXTERN PetscErrorCode PetscMallocAlign(size_t mem,PetscBool clear,int line,const char func[],const char file[],void **result)
     27: {
     28:   if (!mem) {*result = NULL; return 0;}
     29: #if PetscDefined(HAVE_MEMKIND)
     30:   {
     31:     int err;

     33:     err = memkind_posix_memalign(currentmktype ? MEMKIND_HBW_PREFERRED : MEMKIND_DEFAULT,result,PETSC_MEMALIGN,mem);
     35:     if (err == ENOMEM) PetscInfo(NULL,"Memkind: fail to request HBW memory %.0f, falling back to normal memory\n",(PetscLogDouble)mem);
     37:     if (clear) PetscMemzero(*result,mem);
     38:   }
     39: #else /* PetscDefined(HAVE_MEMKIND) */
     40: #  if PetscDefined(HAVE_DOUBLE_ALIGN_MALLOC) && (PETSC_MEMALIGN == 8)
     41:   if (clear) *result = calloc(1+mem/sizeof(int),sizeof(int));
     42:   else       *result = malloc(mem);

     45:   if (PetscLogMemory) PetscMemzero(*result,mem);

     47: #  elif PetscDefined(HAVE_MEMALIGN)
     48:   *result = memalign(PETSC_MEMALIGN,mem);
     50:   if (clear || PetscLogMemory) PetscMemzero(*result,mem);
     51: #  else /* PetscDefined(HAVE_DOUBLE_ALIGN_MALLOC) || PetscDefined(HAVE_MEMALIGN) */
     52:   {
     53:     int *ptr,shift;
     54:     /*
     55:       malloc space for two extra chunks and shift ptr 1 + enough to get it PetscScalar aligned
     56:     */
     57:     if (clear) {
     58:       ptr = (int*)calloc(1+(mem + 2*PETSC_MEMALIGN)/sizeof(int),sizeof(int));
     59:     } else {
     60:       ptr = (int*)malloc(mem + 2*PETSC_MEMALIGN);
     61:     }
     63:     shift        = (int)(((PETSC_UINTPTR_T) ptr) % PETSC_MEMALIGN);
     64:     shift        = (2*PETSC_MEMALIGN - shift)/sizeof(int);
     65:     ptr[shift-1] = shift + SHIFT_CLASSID;
     66:     ptr         += shift;
     67:     *result      = (void*)ptr;
     68:     if (PetscLogMemory) PetscMemzero(*result,mem);
     69:   }
     70: #  endif /* PetscDefined(HAVE_DOUBLE_ALIGN_MALLOC) || PetscDefined(HAVE_MEMALIGN) */
     71: #endif /* PetscDefined(HAVE_MEMKIND) */
     72:   return 0;
     73: }

     75: PETSC_EXTERN PetscErrorCode PetscFreeAlign(void *ptr,int line,const char func[],const char file[])
     76: {
     77:   if (!ptr) return 0;
     78: #if PetscDefined(HAVE_MEMKIND)
     79:   memkind_free(0,ptr); /* specify the kind to 0 so that memkind will look up for the right type */
     80: #else /* PetscDefined(HAVE_MEMKIND) */
     81: #  if (!(PetscDefined(HAVE_DOUBLE_ALIGN_MALLOC) && (PETSC_MEMALIGN == 8)) && !PetscDefined(HAVE_MEMALIGN))
     82:   {
     83:     /*
     84:       Previous int tells us how many ints the pointer has been shifted from
     85:       the original address provided by the system malloc().
     86:     */
     87:     const int shift = *(((int*)ptr)-1) - SHIFT_CLASSID;

     91:     ptr = (void*)(((int*)ptr) - shift);
     92:   }
     93: #  endif

     95: #  if PetscDefined(HAVE_FREE_RETURN_INT)
     96:   int err = free(ptr);
     98: #  else
     99:   free(ptr);
    100: #  endif
    101: #endif
    102:   return 0;
    103: }

    105: PETSC_EXTERN PetscErrorCode PetscReallocAlign(size_t mem, int line, const char func[], const char file[], void **result)
    106: {
    107:   if (!mem) {
    108:     PetscFreeAlign(*result, line, func, file);
    109:     *result = NULL;
    110:     return 0;
    111:   }
    112: #if PetscDefined(HAVE_MEMKIND)
    113:   *result = memkind_realloc(currentmktype ? MEMKIND_HBW_PREFERRED : MEMKIND_DEFAULT,*result,mem);
    114: #else
    115: #  if (!(PetscDefined(HAVE_DOUBLE_ALIGN_MALLOC) && (PETSC_MEMALIGN == 8)) && !PetscDefined(HAVE_MEMALIGN))
    116:   {
    117:     /*
    118:       Previous int tells us how many ints the pointer has been shifted from
    119:       the original address provided by the system malloc().
    120:     */
    121:     int shift = *(((int*)*result)-1) - SHIFT_CLASSID;
    124:     *result = (void*)(((int*)*result) - shift);
    125:   }
    126: #  endif

    128: #  if (PetscDefined(HAVE_DOUBLE_ALIGN_MALLOC) && (PETSC_MEMALIGN == 8)) || PetscDefined(HAVE_MEMALIGN)
    129:   *result = realloc(*result, mem);
    130: #  else
    131:   {
    132:     /*
    133:       malloc space for two extra chunks and shift ptr 1 + enough to get it PetscScalar aligned
    134:     */
    135:     int *ptr = (int*)realloc(*result,mem + 2*PETSC_MEMALIGN);
    136:     if (ptr) {
    137:       int shift    = (int)(((PETSC_UINTPTR_T) ptr) % PETSC_MEMALIGN);
    138:       shift        = (2*PETSC_MEMALIGN - shift)/sizeof(int);
    139:       ptr[shift-1] = shift + SHIFT_CLASSID;
    140:       ptr         += shift;
    141:       *result      = (void*)ptr;
    142:     } else {
    143:       *result      = NULL;
    144:     }
    145:   }
    146: #  endif
    147: #endif
    149: #if PetscDefined(HAVE_MEMALIGN)
    150:   /* There are no standard guarantees that realloc() maintains the alignment of memalign(), so I think we have to
    151:    * realloc and, if the alignment is wrong, malloc/copy/free. */
    152:   if (((size_t) (*result)) % PETSC_MEMALIGN) {
    153:     void *newResult;
    154: #  if PetscDefined(HAVE_MEMKIND)
    155:     {
    156:       int err;
    157:       err = memkind_posix_memalign(currentmktype ? MEMKIND_HBW_PREFERRED : MEMKIND_DEFAULT,&newResult,PETSC_MEMALIGN,mem);
    159:       if (err == ENOMEM) PetscInfo(NULL,"Memkind: fail to request HBW memory %.0f, falling back to normal memory\n",(PetscLogDouble)mem);
    160:     }
    161: #  else
    162:     newResult = memalign(PETSC_MEMALIGN,mem);
    163: #  endif
    165:     PetscMemcpy(newResult,*result,mem);
    166: #  if PetscDefined(HAVE_FREE_RETURN_INT)
    167:     {
    168:       int err = free(*result);
    170:     }
    171: #  else
    172: #    if defined(PETSC_HAVE_MEMKIND)
    173:     memkind_free(0,*result);
    174: #    else
    175:     free(*result);
    176: #    endif
    177: #  endif
    178:     *result = newResult;
    179:   }
    180: #endif
    181:   return 0;
    182: }

    184: PetscErrorCode (*PetscTrMalloc)(size_t,PetscBool,int,const char[],const char[],void**) = PetscMallocAlign;
    185: PetscErrorCode (*PetscTrFree)(void*,int,const char[],const char[])                     = PetscFreeAlign;
    186: PetscErrorCode (*PetscTrRealloc)(size_t,int,const char[],const char[],void**)          = PetscReallocAlign;

    188: PETSC_INTERN PetscBool petscsetmallocvisited;
    189: PetscBool petscsetmallocvisited = PETSC_FALSE;

    191: /*@C
    192:    PetscMallocSet - Sets the routines used to do mallocs and frees.
    193:    This routine MUST be called before PetscInitialize() and may be
    194:    called only once.

    196:    Not Collective

    198:    Input Parameters:
    199: + imalloc - the routine that provides the malloc (also provides calloc(), which is used depends on the second argument)
    200: . ifree - the routine that provides the free
    201: - iralloc - the routine that provides the realloc

    203:    Level: developer

    205: @*/
    206: PetscErrorCode PetscMallocSet(PetscErrorCode (*imalloc)(size_t,PetscBool,int,const char[],const char[],void**),
    207:                               PetscErrorCode (*ifree)(void*,int,const char[],const char[]),
    208:                               PetscErrorCode (*iralloc)(size_t, int, const char[], const char[], void **))
    209: {
    211:   PetscTrMalloc         = imalloc;
    212:   PetscTrFree           = ifree;
    213:   PetscTrRealloc        = iralloc;
    214:   petscsetmallocvisited = PETSC_TRUE;
    215:   return 0;
    216: }

    218: /*@C
    219:    PetscMallocClear - Resets the routines used to do mallocs and frees to the defaults.

    221:    Not Collective

    223:    Level: developer

    225:    Notes:
    226:     In general one should never run a PETSc program with different malloc() and
    227:     free() settings for different parts; this is because one NEVER wants to
    228:     free() an address that was malloced by a different memory management system

    230:     Called in PetscFinalize() so that if PetscInitialize() is called again it starts with a fresh slate of allocation information

    232: @*/
    233: PetscErrorCode PetscMallocClear(void)
    234: {
    235:   PetscTrMalloc         = PetscMallocAlign;
    236:   PetscTrFree           = PetscFreeAlign;
    237:   PetscTrRealloc        = PetscReallocAlign;
    238:   petscsetmallocvisited = PETSC_FALSE;
    239:   return 0;
    240: }

    242: PetscErrorCode PetscMemoryTrace(const char label[])
    243: {
    244:   PetscLogDouble        mem,mal;
    245:   static PetscLogDouble oldmem = 0,oldmal = 0;

    247:   PetscMemoryGetCurrentUsage(&mem);
    248:   PetscMallocGetCurrentUsage(&mal);

    250:   PetscPrintf(PETSC_COMM_WORLD,"%s High water  %8.3f MB increase %8.3f MB Current %8.3f MB increase %8.3f MB\n",label,mem*1e-6,(mem - oldmem)*1e-6,mal*1e-6,(mal - oldmal)*1e-6);
    251:   oldmem = mem;
    252:   oldmal = mal;
    253:   return 0;
    254: }

    256: static PetscErrorCode (*PetscTrMallocOld)(size_t,PetscBool,int,const char[],const char[],void**) = PetscMallocAlign;
    257: static PetscErrorCode (*PetscTrReallocOld)(size_t,int,const char[],const char[],void**)          = PetscReallocAlign;
    258: static PetscErrorCode (*PetscTrFreeOld)(void*,int,const char[],const char[])                     = PetscFreeAlign;

    260: /*@C
    261:    PetscMallocSetDRAM - Set PetscMalloc to use DRAM.
    262:      If memkind is available, change the memkind type. Otherwise, switch the
    263:      current malloc and free routines to the PetscMallocAlign and
    264:      PetscFreeAlign (PETSc default).

    266:    Not Collective

    268:    Level: developer

    270:    Notes:
    271:      This provides a way to do the allocation on DRAM temporarily. One
    272:      can switch back to the previous choice by calling PetscMallocReset().

    274: .seealso: PetscMallocReset()
    275: @*/
    276: PetscErrorCode PetscMallocSetDRAM(void)
    277: {
    278:   if (PetscTrMalloc == PetscMallocAlign) {
    279: #if defined(PETSC_HAVE_MEMKIND)
    280:     previousmktype = currentmktype;
    281:     currentmktype  = PETSC_MK_DEFAULT;
    282: #endif
    283:   } else {
    284:     /* Save the previous choice */
    285:     PetscTrMallocOld  = PetscTrMalloc;
    286:     PetscTrReallocOld = PetscTrRealloc;
    287:     PetscTrFreeOld    = PetscTrFree;
    288:     PetscTrMalloc     = PetscMallocAlign;
    289:     PetscTrFree       = PetscFreeAlign;
    290:     PetscTrRealloc    = PetscReallocAlign;
    291:   }
    292:   return 0;
    293: }

    295: /*@C
    296:    PetscMallocResetDRAM - Reset the changes made by PetscMallocSetDRAM

    298:    Not Collective

    300:    Level: developer

    302: .seealso: PetscMallocSetDRAM()
    303: @*/
    304: PetscErrorCode PetscMallocResetDRAM(void)
    305: {
    306:   if (PetscTrMalloc == PetscMallocAlign) {
    307: #if defined(PETSC_HAVE_MEMKIND)
    308:     currentmktype = previousmktype;
    309: #endif
    310:   } else {
    311:     /* Reset to the previous choice */
    312:     PetscTrMalloc  = PetscTrMallocOld;
    313:     PetscTrRealloc = PetscTrReallocOld;
    314:     PetscTrFree    = PetscTrFreeOld;
    315:   }
    316:   return 0;
    317: }

    319: static PetscBool petscmalloccoalesce =
    320: #if defined(PETSC_USE_MALLOC_COALESCED)
    321:   PETSC_TRUE;
    322: #else
    323:   PETSC_FALSE;
    324: #endif

    326: /*@C
    327:    PetscMallocSetCoalesce - Use coalesced malloc when allocating groups of objects

    329:    Not Collective

    331:    Input Parameters:
    332: .  coalesce - PETSC_TRUE to use coalesced malloc for multi-object allocation.

    334:    Options Database Keys:
    335: .  -malloc_coalesce - turn coalesced malloc on or off

    337:    Note:
    338:    PETSc uses coalesced malloc by default for optimized builds and not for debugging builds.  This default can be changed via the command-line option -malloc_coalesce or by calling this function.
    339:    This function can only be called immediately after PetscInitialize()

    341:    Level: developer

    343: .seealso: PetscMallocA()
    344: @*/
    345: PetscErrorCode PetscMallocSetCoalesce(PetscBool coalesce)
    346: {
    347:   petscmalloccoalesce = coalesce;
    348:   return 0;
    349: }

    351: /*@C
    352:    PetscMallocA - Allocate and optionally clear one or more objects, possibly using coalesced malloc

    354:    Not Collective

    356:    Input Parameters:
    357: +  n - number of objects to allocate (at least 1)
    358: .  clear - use calloc() to allocate space initialized to zero
    359: .  lineno - line number to attribute allocation (typically __LINE__)
    360: .  function - function to attribute allocation (typically PETSC_FUNCTION_NAME)
    361: .  filename - file name to attribute allocation (typically __FILE__)
    362: -  bytes0 - first of n object sizes

    364:    Output Parameters:
    365: .  ptr0 - first of n pointers to allocate

    367:    Notes:
    368:    This function is not normally called directly by users, but rather via the macros PetscMalloc1(), PetscMalloc2(), or PetscCalloc1(), etc.

    370:    Level: developer

    372: .seealso: PetscMallocAlign(), PetscMallocSet(), PetscMalloc1(), PetscMalloc2(), PetscMalloc3(), PetscMalloc4(), PetscMalloc5(), PetscMalloc6(), PetscMalloc7(), PetscCalloc1(), PetscCalloc2(), PetscCalloc3(), PetscCalloc4(), PetscCalloc5(), PetscCalloc6(), PetscCalloc7(), PetscFreeA()
    373: @*/
    374: PetscErrorCode PetscMallocA(int n,PetscBool clear,int lineno,const char *function,const char *filename,size_t bytes0,void *ptr0,...)
    375: {
    376:   va_list        Argp;
    377:   size_t         bytes[8],sumbytes;
    378:   void           **ptr[8];
    379:   int            i;

    382:   bytes[0] = bytes0;
    383:   ptr[0] = (void**)ptr0;
    384:   sumbytes = (bytes0 + PETSC_MEMALIGN-1) & ~(PETSC_MEMALIGN-1);
    385:   va_start(Argp,ptr0);
    386:   for (i=1; i<n; i++) {
    387:     bytes[i] = va_arg(Argp,size_t);
    388:     ptr[i] = va_arg(Argp,void**);
    389:     sumbytes += (bytes[i] + PETSC_MEMALIGN-1) & ~(PETSC_MEMALIGN-1);
    390:   }
    391:   va_end(Argp);
    392:   if (petscmalloccoalesce) {
    393:     char *p;
    394:     (*PetscTrMalloc)(sumbytes,clear,lineno,function,filename,(void**)&p);
    395:     for (i=0; i<n; i++) {
    396:       *ptr[i] = bytes[i] ? p : NULL;
    397:       p = (char*)PetscAddrAlign(p + bytes[i]);
    398:     }
    399:   } else {
    400:     for (i=0; i<n; i++) {
    401:       (*PetscTrMalloc)(bytes[i],clear,lineno,function,filename,(void**)ptr[i]);
    402:     }
    403:   }
    404:   return 0;
    405: }

    407: /*@C
    408:    PetscFreeA - Free one or more objects, possibly allocated using coalesced malloc

    410:    Not Collective

    412:    Input Parameters:
    413: +  n - number of objects to free (at least 1)
    414: .  lineno - line number to attribute deallocation (typically __LINE__)
    415: .  function - function to attribute deallocation (typically PETSC_FUNCTION_NAME)
    416: .  filename - file name to attribute deallocation (typically __FILE__)
    417: -  ptr0 ... - first of n pointers to free

    419:    Note:
    420:    This function is not normally called directly by users, but rather via the macros PetscFree(), PetscFree2(), etc.

    422:    The pointers are zeroed to prevent users from accidentally reusing space that has been freed.

    424:    Level: developer

    426: .seealso: PetscMallocAlign(), PetscMallocSet(), PetscMallocA(), PetscFree1(), PetscFree2(), PetscFree3(), PetscFree4(), PetscFree5(), PetscFree6(), PetscFree7()
    427: @*/
    428: PetscErrorCode PetscFreeA(int n,int lineno,const char *function,const char *filename,void *ptr0,...)
    429: {
    430:   va_list   Argp;
    431:   void    **ptr[8];
    432:   int       i;

    435:   ptr[0] = (void**)ptr0;
    436:   va_start(Argp,ptr0);
    437:   for (i=1; i<n; i++) {
    438:     ptr[i] = va_arg(Argp,void**);
    439:   }
    440:   va_end(Argp);
    441:   if (petscmalloccoalesce) {
    442:     for (i=0; i<n; i++) {       /* Find first nonempty allocation */
    443:       if (*ptr[i]) break;
    444:     }
    445:     while (--n > i) {
    446:       *ptr[n] = NULL;
    447:     }
    448:     (*PetscTrFree)(*ptr[n],lineno,function,filename);
    449:     *ptr[n] = NULL;
    450:   } else {
    451:     while (--n >= 0) {
    452:       (*PetscTrFree)(*ptr[n],lineno,function,filename);
    453:       *ptr[n] = NULL;
    454:     }
    455:   }
    456:   return 0;
    457: }