Date: 2022-03-31T04:46:41+00:00

Actual source code: isltog.c

      2: #include <petsc/private/isimpl.h>
      3: #include <petsc/private/hashmapi.h>
      4: #include <petscsf.h>
      5: #include <petscviewer.h>

      7: PetscClassId IS_LTOGM_CLASSID;
      8: static PetscErrorCode  ISLocalToGlobalMappingGetBlockInfo_Private(ISLocalToGlobalMapping,PetscInt*,PetscInt**,PetscInt**,PetscInt***);

     10: typedef struct {
     11:   PetscInt *globals;
     12: } ISLocalToGlobalMapping_Basic;

     14: typedef struct {
     15:   PetscHMapI globalht;
     16: } ISLocalToGlobalMapping_Hash;

     18: /*@C
     19:   ISGetPointRange - Returns a description of the points in an IS suitable for traversal

     21:   Not collective

     23:   Input Parameter:
     24: . pointIS - The IS object

     26:   Output Parameters:
     27: + pStart - The first index, see notes
     28: . pEnd   - One past the last index, see notes
     29: - points - The indices, see notes

     31:   Notes:
     32:   If the IS contains contiguous indices in an ISSTRIDE, then the indices are contained in [pStart, pEnd) and points = NULL. Otherwise, pStart = 0, pEnd = numIndices, and points is an array of the indices. This supports the following pattern
     33: $ ISGetPointRange(is, &pStart, &pEnd, &points);
     34: $ for (p = pStart; p < pEnd; ++p) {
     35: $   const PetscInt point = points ? points[p] : p;
     36: $ }
     37: $ ISRestorePointRange(is, &pstart, &pEnd, &points);

     39:   Level: intermediate

     41: .seealso: ISRestorePointRange(), ISGetPointSubrange(), ISGetIndices(), ISCreateStride()
     42: @*/
     43: PetscErrorCode ISGetPointRange(IS pointIS, PetscInt *pStart, PetscInt *pEnd, const PetscInt **points)
     44: {
     45:   PetscInt       numCells, step = 1;
     46:   PetscBool      isStride;

     49:   *pStart = 0;
     50:   *points = NULL;
     51:   ISGetLocalSize(pointIS, &numCells);
     52:   PetscObjectTypeCompare((PetscObject) pointIS, ISSTRIDE, &isStride);
     53:   if (isStride) ISStrideGetInfo(pointIS, pStart, &step);
     54:   *pEnd   = *pStart + numCells;
     55:   if (!isStride || step != 1) ISGetIndices(pointIS, points);
     56:   return 0;
     57: }

     59: /*@C
     60:   ISRestorePointRange - Destroys the traversal description

     62:   Not collective

     64:   Input Parameters:
     65: + pointIS - The IS object
     66: . pStart  - The first index, from ISGetPointRange()
     67: . pEnd    - One past the last index, from ISGetPointRange()
     68: - points  - The indices, from ISGetPointRange()

     70:   Notes:
     71:   If the IS contains contiguous indices in an ISSTRIDE, then the indices are contained in [pStart, pEnd) and points = NULL. Otherwise, pStart = 0, pEnd = numIndices, and points is an array of the indices. This supports the following pattern
     72: $ ISGetPointRange(is, &pStart, &pEnd, &points);
     73: $ for (p = pStart; p < pEnd; ++p) {
     74: $   const PetscInt point = points ? points[p] : p;
     75: $ }
     76: $ ISRestorePointRange(is, &pstart, &pEnd, &points);

     78:   Level: intermediate

     80: .seealso: ISGetPointRange(), ISGetPointSubrange(), ISGetIndices(), ISCreateStride()
     81: @*/
     82: PetscErrorCode ISRestorePointRange(IS pointIS, PetscInt *pStart, PetscInt *pEnd, const PetscInt **points)
     83: {
     84:   PetscInt       step = 1;
     85:   PetscBool      isStride;

     88:   PetscObjectTypeCompare((PetscObject) pointIS, ISSTRIDE, &isStride);
     89:   if (isStride) ISStrideGetInfo(pointIS, pStart, &step);
     90:   if (!isStride || step != 1) ISGetIndices(pointIS, points);
     91:   return 0;
     92: }

     94: /*@C
     95:   ISGetPointSubrange - Configures the input IS to be a subrange for the traversal information given

     97:   Not collective

     99:   Input Parameters:
    100: + subpointIS - The IS object to be configured
    101: . pStar   t  - The first index of the subrange
    102: . pEnd       - One past the last index for the subrange
    103: - points     - The indices for the entire range, from ISGetPointRange()

    105:   Output Parameters:
    106: . subpointIS - The IS object now configured to be a subrange

    108:   Notes:
    109:   The input IS will now respond properly to calls to ISGetPointRange() and return the subrange.

    111:   Level: intermediate

    113: .seealso: ISGetPointRange(), ISRestorePointRange(), ISGetIndices(), ISCreateStride()
    114: @*/
    115: PetscErrorCode ISGetPointSubrange(IS subpointIS, PetscInt pStart, PetscInt pEnd, const PetscInt *points)
    116: {
    118:   if (points) {
    119:     ISSetType(subpointIS, ISGENERAL);
    120:     ISGeneralSetIndices(subpointIS, pEnd-pStart, &points[pStart], PETSC_USE_POINTER);
    121:   } else {
    122:     ISSetType(subpointIS, ISSTRIDE);
    123:     ISStrideSetStride(subpointIS, pEnd-pStart, pStart, 1);
    124:   }
    125:   return 0;
    126: }

    128: /* -----------------------------------------------------------------------------------------*/

    130: /*
    131:     Creates the global mapping information in the ISLocalToGlobalMapping structure

    133:     If the user has not selected how to handle the global to local mapping then use HASH for "large" problems
    134: */
    135: static PetscErrorCode ISGlobalToLocalMappingSetUp(ISLocalToGlobalMapping mapping)
    136: {
    137:   PetscInt       i,*idx = mapping->indices,n = mapping->n,end,start;

    139:   if (mapping->data) return 0;
    140:   end   = 0;
    141:   start = PETSC_MAX_INT;

    143:   for (i=0; i<n; i++) {
    144:     if (idx[i] < 0) continue;
    145:     if (idx[i] < start) start = idx[i];
    146:     if (idx[i] > end)   end   = idx[i];
    147:   }
    148:   if (start > end) {start = 0; end = -1;}
    149:   mapping->globalstart = start;
    150:   mapping->globalend   = end;
    151:   if (!((PetscObject)mapping)->type_name) {
    152:     if ((end - start) > PetscMax(4*n,1000000)) {
    153:       ISLocalToGlobalMappingSetType(mapping,ISLOCALTOGLOBALMAPPINGHASH);
    154:     } else {
    155:       ISLocalToGlobalMappingSetType(mapping,ISLOCALTOGLOBALMAPPINGBASIC);
    156:     }
    157:   }
    158:   if (mapping->ops->globaltolocalmappingsetup) (*mapping->ops->globaltolocalmappingsetup)(mapping);
    159:   return 0;
    160: }

    162: static PetscErrorCode ISGlobalToLocalMappingSetUp_Basic(ISLocalToGlobalMapping mapping)
    163: {
    164:   PetscInt                    i,*idx = mapping->indices,n = mapping->n,end,start,*globals;
    165:   ISLocalToGlobalMapping_Basic *map;

    167:   start            = mapping->globalstart;
    168:   end              = mapping->globalend;
    169:   PetscNew(&map);
    170:   PetscMalloc1(end-start+2,&globals);
    171:   map->globals     = globals;
    172:   for (i=0; i<end-start+1; i++) globals[i] = -1;
    173:   for (i=0; i<n; i++) {
    174:     if (idx[i] < 0) continue;
    175:     globals[idx[i] - start] = i;
    176:   }
    177:   mapping->data = (void*)map;
    178:   PetscLogObjectMemory((PetscObject)mapping,(end-start+1)*sizeof(PetscInt));
    179:   return 0;
    180: }

    182: static PetscErrorCode ISGlobalToLocalMappingSetUp_Hash(ISLocalToGlobalMapping mapping)
    183: {
    184:   PetscInt                    i,*idx = mapping->indices,n = mapping->n;
    185:   ISLocalToGlobalMapping_Hash *map;

    187:   PetscNew(&map);
    188:   PetscHMapICreate(&map->globalht);
    189:   for (i=0; i<n; i++) {
    190:     if (idx[i] < 0) continue;
    191:     PetscHMapISet(map->globalht,idx[i],i);
    192:   }
    193:   mapping->data = (void*)map;
    194:   PetscLogObjectMemory((PetscObject)mapping,2*n*sizeof(PetscInt));
    195:   return 0;
    196: }

    198: static PetscErrorCode ISLocalToGlobalMappingDestroy_Basic(ISLocalToGlobalMapping mapping)
    199: {
    200:   ISLocalToGlobalMapping_Basic *map  = (ISLocalToGlobalMapping_Basic *)mapping->data;

    202:   if (!map) return 0;
    203:   PetscFree(map->globals);
    204:   PetscFree(mapping->data);
    205:   return 0;
    206: }

    208: static PetscErrorCode ISLocalToGlobalMappingDestroy_Hash(ISLocalToGlobalMapping mapping)
    209: {
    210:   ISLocalToGlobalMapping_Hash *map  = (ISLocalToGlobalMapping_Hash*)mapping->data;

    212:   if (!map) return 0;
    213:   PetscHMapIDestroy(&map->globalht);
    214:   PetscFree(mapping->data);
    215:   return 0;
    216: }

    218: #define GTOLTYPE _Basic
    219: #define GTOLNAME _Basic
    220: #define GTOLBS mapping->bs
    221: #define GTOL(g, local) do {                  \
    222:     local = map->globals[g/bs - start];      \
    223:     if (local >= 0) local = bs*local + (g % bs); \
    224:   } while (0)

    226: #include <../src/vec/is/utils/isltog.h>

    228: #define GTOLTYPE _Basic
    229: #define GTOLNAME Block_Basic
    230: #define GTOLBS 1
    231: #define GTOL(g, local) do {                  \
    232:     local = map->globals[g - start];         \
    233:   } while (0)
    234: #include <../src/vec/is/utils/isltog.h>

    236: #define GTOLTYPE _Hash
    237: #define GTOLNAME _Hash
    238: #define GTOLBS mapping->bs
    239: #define GTOL(g, local) do {                         \
    240:     (void)PetscHMapIGet(map->globalht,g/bs,&local); \
    241:     if (local >= 0) local = bs*local + (g % bs);   \
    242:    } while (0)
    243: #include <../src/vec/is/utils/isltog.h>

    245: #define GTOLTYPE _Hash
    246: #define GTOLNAME Block_Hash
    247: #define GTOLBS 1
    248: #define GTOL(g, local) do {                         \
    249:     (void)PetscHMapIGet(map->globalht,g,&local);    \
    250:   } while (0)
    251: #include <../src/vec/is/utils/isltog.h>

    253: /*@
    254:     ISLocalToGlobalMappingDuplicate - Duplicates the local to global mapping object

    256:     Not Collective

    258:     Input Parameter:
    259: .   ltog - local to global mapping

    261:     Output Parameter:
    262: .   nltog - the duplicated local to global mapping

    264:     Level: advanced

    266: .seealso: ISLocalToGlobalMappingDestroy(), ISLocalToGlobalMappingCreate()
    267: @*/
    268: PetscErrorCode  ISLocalToGlobalMappingDuplicate(ISLocalToGlobalMapping ltog,ISLocalToGlobalMapping* nltog)
    269: {
    270:   ISLocalToGlobalMappingType l2gtype;

    273:   ISLocalToGlobalMappingCreate(PetscObjectComm((PetscObject)ltog),ltog->bs,ltog->n,ltog->indices,PETSC_COPY_VALUES,nltog);
    274:   ISLocalToGlobalMappingGetType(ltog,&l2gtype);
    275:   ISLocalToGlobalMappingSetType(*nltog,l2gtype);
    276:   return 0;
    277: }

    279: /*@
    280:     ISLocalToGlobalMappingGetSize - Gets the local size of a local to global mapping

    282:     Not Collective

    284:     Input Parameter:
    285: .   ltog - local to global mapping

    287:     Output Parameter:
    288: .   n - the number of entries in the local mapping, ISLocalToGlobalMappingGetIndices() returns an array of this length

    290:     Level: advanced

    292: .seealso: ISLocalToGlobalMappingDestroy(), ISLocalToGlobalMappingCreate()
    293: @*/
    294: PetscErrorCode  ISLocalToGlobalMappingGetSize(ISLocalToGlobalMapping mapping,PetscInt *n)
    295: {
    298:   *n = mapping->bs*mapping->n;
    299:   return 0;
    300: }

    302: /*@C
    303:    ISLocalToGlobalMappingViewFromOptions - View from Options

    305:    Collective on ISLocalToGlobalMapping

    307:    Input Parameters:
    308: +  A - the local to global mapping object
    309: .  obj - Optional object
    310: -  name - command line option

    312:    Level: intermediate
    313: .seealso:  ISLocalToGlobalMapping, ISLocalToGlobalMappingView, PetscObjectViewFromOptions(), ISLocalToGlobalMappingCreate()
    314: @*/
    315: PetscErrorCode  ISLocalToGlobalMappingViewFromOptions(ISLocalToGlobalMapping A,PetscObject obj,const char name[])
    316: {
    318:   PetscObjectViewFromOptions((PetscObject)A,obj,name);
    319:   return 0;
    320: }

    322: /*@C
    323:     ISLocalToGlobalMappingView - View a local to global mapping

    325:     Not Collective

    327:     Input Parameters:
    328: +   ltog - local to global mapping
    329: -   viewer - viewer

    331:     Level: advanced

    333: .seealso: ISLocalToGlobalMappingDestroy(), ISLocalToGlobalMappingCreate()
    334: @*/
    335: PetscErrorCode  ISLocalToGlobalMappingView(ISLocalToGlobalMapping mapping,PetscViewer viewer)
    336: {
    337:   PetscInt       i;
    338:   PetscMPIInt    rank;
    339:   PetscBool      iascii;

    342:   if (!viewer) {
    343:     PetscViewerASCIIGetStdout(PetscObjectComm((PetscObject)mapping),&viewer);
    344:   }

    347:   MPI_Comm_rank(PetscObjectComm((PetscObject)mapping),&rank);
    348:   PetscObjectTypeCompare((PetscObject)viewer,PETSCVIEWERASCII,&iascii);
    349:   if (iascii) {
    350:     PetscObjectPrintClassNamePrefixType((PetscObject)mapping,viewer);
    351:     PetscViewerASCIIPushSynchronized(viewer);
    352:     for (i=0; i<mapping->n; i++) {
    353:       PetscViewerASCIISynchronizedPrintf(viewer,"[%d] %" PetscInt_FMT " %" PetscInt_FMT "\n",rank,i,mapping->indices[i]);
    354:     }
    355:     PetscViewerFlush(viewer);
    356:     PetscViewerASCIIPopSynchronized(viewer);
    357:   }
    358:   return 0;
    359: }

    361: /*@
    362:     ISLocalToGlobalMappingCreateIS - Creates a mapping between a local (0 to n)
    363:     ordering and a global parallel ordering.

    365:     Not collective

    367:     Input Parameter:
    368: .   is - index set containing the global numbers for each local number

    370:     Output Parameter:
    371: .   mapping - new mapping data structure

    373:     Notes:
    374:     the block size of the IS determines the block size of the mapping
    375:     Level: advanced

    377: .seealso: ISLocalToGlobalMappingDestroy(), ISLocalToGlobalMappingCreate(), ISLocalToGlobalMappingSetFromOptions()
    378: @*/
    379: PetscErrorCode  ISLocalToGlobalMappingCreateIS(IS is,ISLocalToGlobalMapping *mapping)
    380: {
    381:   PetscInt       n,bs;
    382:   const PetscInt *indices;
    383:   MPI_Comm       comm;
    384:   PetscBool      isblock;

    389:   PetscObjectGetComm((PetscObject)is,&comm);
    390:   ISGetLocalSize(is,&n);
    391:   PetscObjectTypeCompare((PetscObject)is,ISBLOCK,&isblock);
    392:   if (!isblock) {
    393:     ISGetIndices(is,&indices);
    394:     ISLocalToGlobalMappingCreate(comm,1,n,indices,PETSC_COPY_VALUES,mapping);
    395:     ISRestoreIndices(is,&indices);
    396:   } else {
    397:     ISGetBlockSize(is,&bs);
    398:     ISBlockGetIndices(is,&indices);
    399:     ISLocalToGlobalMappingCreate(comm,bs,n/bs,indices,PETSC_COPY_VALUES,mapping);
    400:     ISBlockRestoreIndices(is,&indices);
    401:   }
    402:   return 0;
    403: }

    405: /*@C
    406:     ISLocalToGlobalMappingCreateSF - Creates a mapping between a local (0 to n)
    407:     ordering and a global parallel ordering.

    409:     Collective

    411:     Input Parameters:
    412: +   sf - star forest mapping contiguous local indices to (rank, offset)
    413: -   start - first global index on this process, or PETSC_DECIDE to compute contiguous global numbering automatically

    415:     Output Parameter:
    416: .   mapping - new mapping data structure

    418:     Level: advanced

    420:     Notes:
    421:     If any processor calls this with start = PETSC_DECIDE then all processors must, otherwise the program will hang.

    423: .seealso: ISLocalToGlobalMappingDestroy(), ISLocalToGlobalMappingCreate(), ISLocalToGlobalMappingCreateIS(), ISLocalToGlobalMappingSetFromOptions()
    424: @*/
    425: PetscErrorCode ISLocalToGlobalMappingCreateSF(PetscSF sf,PetscInt start,ISLocalToGlobalMapping *mapping)
    426: {
    427:   PetscInt       i,maxlocal,nroots,nleaves,*globals,*ltog;
    428:   const PetscInt *ilocal;
    429:   MPI_Comm       comm;

    434:   PetscObjectGetComm((PetscObject)sf,&comm);
    435:   PetscSFGetGraph(sf,&nroots,&nleaves,&ilocal,NULL);
    436:   if (start == PETSC_DECIDE) {
    437:     start = 0;
    438:     MPI_Exscan(&nroots,&start,1,MPIU_INT,MPI_SUM,comm);
    440:   if (ilocal) {
    441:     for (i=0,maxlocal=0; i<nleaves; i++) maxlocal = PetscMax(maxlocal,ilocal[i]+1);
    442:   }
    443:   else maxlocal = nleaves;
    444:   PetscMalloc1(nroots,&globals);
    445:   PetscMalloc1(maxlocal,&ltog);
    446:   for (i=0; i<nroots; i++) globals[i] = start + i;
    447:   for (i=0; i<maxlocal; i++) ltog[i] = -1;
    448:   PetscSFBcastBegin(sf,MPIU_INT,globals,ltog,MPI_REPLACE);
    449:   PetscSFBcastEnd(sf,MPIU_INT,globals,ltog,MPI_REPLACE);
    450:   ISLocalToGlobalMappingCreate(comm,1,maxlocal,ltog,PETSC_OWN_POINTER,mapping);
    451:   PetscFree(globals);
    452:   return 0;
    453: }

    455: /*@
    456:     ISLocalToGlobalMappingSetBlockSize - Sets the blocksize of the mapping

    458:     Not collective

    460:     Input Parameters:
    461: +   mapping - mapping data structure
    462: -   bs - the blocksize

    464:     Level: advanced

    466: .seealso: ISLocalToGlobalMappingDestroy(), ISLocalToGlobalMappingCreateIS()
    467: @*/
    468: PetscErrorCode  ISLocalToGlobalMappingSetBlockSize(ISLocalToGlobalMapping mapping,PetscInt bs)
    469: {
    470:   PetscInt       *nid;
    471:   const PetscInt *oid;
    472:   PetscInt       i,cn,on,obs,nn;

    476:   if (bs == mapping->bs) return 0;
    477:   on  = mapping->n;
    478:   obs = mapping->bs;
    479:   oid = mapping->indices;
    480:   nn  = (on*obs)/bs;

    483:   PetscMalloc1(nn,&nid);
    484:   ISLocalToGlobalMappingGetIndices(mapping,&oid);
    485:   for (i=0;i<nn;i++) {
    486:     PetscInt j;
    487:     for (j=0,cn=0;j<bs-1;j++) {
    488:       if (oid[i*bs+j] < 0) { cn++; continue; }
    490:     }
    491:     if (oid[i*bs+j] < 0) cn++;
    492:     if (cn) {
    494:       nid[i] = -1;
    495:     } else {
    496:       nid[i] = oid[i*bs]/bs;
    497:     }
    498:   }
    499:   ISLocalToGlobalMappingRestoreIndices(mapping,&oid);

    501:   mapping->n           = nn;
    502:   mapping->bs          = bs;
    503:   PetscFree(mapping->indices);
    504:   mapping->indices     = nid;
    505:   mapping->globalstart = 0;
    506:   mapping->globalend   = 0;

    508:   /* reset the cached information */
    509:   PetscFree(mapping->info_procs);
    510:   PetscFree(mapping->info_numprocs);
    511:   if (mapping->info_indices) {
    512:     PetscInt i;

    514:     PetscFree((mapping->info_indices)[0]);
    515:     for (i=1; i<mapping->info_nproc; i++) {
    516:       PetscFree(mapping->info_indices[i]);
    517:     }
    518:     PetscFree(mapping->info_indices);
    519:   }
    520:   mapping->info_cached = PETSC_FALSE;

    522:   if (mapping->ops->destroy) {
    523:     (*mapping->ops->destroy)(mapping);
    524:   }
    525:   return 0;
    526: }

    528: /*@
    529:     ISLocalToGlobalMappingGetBlockSize - Gets the blocksize of the mapping
    530:     ordering and a global parallel ordering.

    532:     Not Collective

    534:     Input Parameters:
    535: .   mapping - mapping data structure

    537:     Output Parameter:
    538: .   bs - the blocksize

    540:     Level: advanced

    542: .seealso: ISLocalToGlobalMappingDestroy(), ISLocalToGlobalMappingCreateIS()
    543: @*/
    544: PetscErrorCode  ISLocalToGlobalMappingGetBlockSize(ISLocalToGlobalMapping mapping,PetscInt *bs)
    545: {
    547:   *bs = mapping->bs;
    548:   return 0;
    549: }

    551: /*@
    552:     ISLocalToGlobalMappingCreate - Creates a mapping between a local (0 to n)
    553:     ordering and a global parallel ordering.

    555:     Not Collective, but communicator may have more than one process

    557:     Input Parameters:
    558: +   comm - MPI communicator
    559: .   bs - the block size
    560: .   n - the number of local elements divided by the block size, or equivalently the number of block indices
    561: .   indices - the global index for each local element, these do not need to be in increasing order (sorted), these values should not be scaled (i.e. multiplied) by the blocksize bs
    562: -   mode - see PetscCopyMode

    564:     Output Parameter:
    565: .   mapping - new mapping data structure

    567:     Notes:
    568:     There is one integer value in indices per block and it represents the actual indices bs*idx + j, where j=0,..,bs-1

    570:     For "small" problems when using ISGlobalToLocalMappingApply() and ISGlobalToLocalMappingApplyBlock(), the ISLocalToGlobalMappingType of ISLOCALTOGLOBALMAPPINGBASIC will be used;
    571:     this uses more memory but is faster; this approach is not scalable for extremely large mappings. For large problems ISLOCALTOGLOBALMAPPINGHASH is used, this is scalable.
    572:     Use ISLocalToGlobalMappingSetType() or call ISLocalToGlobalMappingSetFromOptions() with the option -islocaltoglobalmapping_type <basic,hash> to control which is used.

    574:     Level: advanced

    576: .seealso: ISLocalToGlobalMappingDestroy(), ISLocalToGlobalMappingCreateIS(), ISLocalToGlobalMappingSetFromOptions(), ISLOCALTOGLOBALMAPPINGBASIC, ISLOCALTOGLOBALMAPPINGHASH
    577:           ISLocalToGlobalMappingSetType(), ISLocalToGlobalMappingType
    578: @*/
    579: PetscErrorCode  ISLocalToGlobalMappingCreate(MPI_Comm comm,PetscInt bs,PetscInt n,const PetscInt indices[],PetscCopyMode mode,ISLocalToGlobalMapping *mapping)
    580: {
    581:   PetscInt       *in;

    586:   *mapping = NULL;
    587:   ISInitializePackage();

    589:   PetscHeaderCreate(*mapping,IS_LTOGM_CLASSID,"ISLocalToGlobalMapping","Local to global mapping","IS",comm,ISLocalToGlobalMappingDestroy,ISLocalToGlobalMappingView);
    590:   (*mapping)->n  = n;
    591:   (*mapping)->bs = bs;
    592:   if (mode == PETSC_COPY_VALUES) {
    593:     PetscMalloc1(n,&in);
    594:     PetscArraycpy(in,indices,n);
    595:     (*mapping)->indices = in;
    596:     (*mapping)->dealloc_indices = PETSC_TRUE;
    597:     PetscLogObjectMemory((PetscObject)*mapping,n*sizeof(PetscInt));
    598:   } else if (mode == PETSC_OWN_POINTER) {
    599:     (*mapping)->indices = (PetscInt*)indices;
    600:     (*mapping)->dealloc_indices = PETSC_TRUE;
    601:     PetscLogObjectMemory((PetscObject)*mapping,n*sizeof(PetscInt));
    602:   } else if (mode == PETSC_USE_POINTER) {
    603:     (*mapping)->indices = (PetscInt*)indices;
    604:   }
    605:   else SETERRQ(comm,PETSC_ERR_ARG_OUTOFRANGE,"Invalid mode %d", mode);
    606:   return 0;
    607: }

    609: PetscFunctionList ISLocalToGlobalMappingList = NULL;

    611: /*@
    612:    ISLocalToGlobalMappingSetFromOptions - Set mapping options from the options database.

    614:    Not collective

    616:    Input Parameters:
    617: .  mapping - mapping data structure

    619:    Level: advanced

    621: @*/
    622: PetscErrorCode ISLocalToGlobalMappingSetFromOptions(ISLocalToGlobalMapping mapping)
    623: {
    624:   PetscErrorCode             ierr;
    625:   char                       type[256];
    626:   ISLocalToGlobalMappingType defaulttype = "Not set";
    627:   PetscBool                  flg;

    630:   ISLocalToGlobalMappingRegisterAll();
    631:   PetscObjectOptionsBegin((PetscObject)mapping);
    632:   PetscOptionsFList("-islocaltoglobalmapping_type","ISLocalToGlobalMapping method","ISLocalToGlobalMappingSetType",ISLocalToGlobalMappingList,(char*)(((PetscObject)mapping)->type_name) ? ((PetscObject)mapping)->type_name : defaulttype,type,256,&flg);
    633:   if (flg) {
    634:     ISLocalToGlobalMappingSetType(mapping,type);
    635:   }
    636:   PetscOptionsEnd();
    637:   return 0;
    638: }

    640: /*@
    641:    ISLocalToGlobalMappingDestroy - Destroys a mapping between a local (0 to n)
    642:    ordering and a global parallel ordering.

    644:    Note Collective

    646:    Input Parameters:
    647: .  mapping - mapping data structure

    649:    Level: advanced

    651: .seealso: ISLocalToGlobalMappingCreate()
    652: @*/
    653: PetscErrorCode  ISLocalToGlobalMappingDestroy(ISLocalToGlobalMapping *mapping)
    654: {
    655:   if (!*mapping) return 0;
    657:   if (--((PetscObject)(*mapping))->refct > 0) {*mapping = NULL;return 0;}
    658:   if ((*mapping)->dealloc_indices) {
    659:     PetscFree((*mapping)->indices);
    660:   }
    661:   PetscFree((*mapping)->info_procs);
    662:   PetscFree((*mapping)->info_numprocs);
    663:   if ((*mapping)->info_indices) {
    664:     PetscInt i;

    666:     PetscFree(((*mapping)->info_indices)[0]);
    667:     for (i=1; i<(*mapping)->info_nproc; i++) {
    668:       PetscFree(((*mapping)->info_indices)[i]);
    669:     }
    670:     PetscFree((*mapping)->info_indices);
    671:   }
    672:   if ((*mapping)->info_nodei) {
    673:     PetscFree(((*mapping)->info_nodei)[0]);
    674:   }
    675:   PetscFree2((*mapping)->info_nodec,(*mapping)->info_nodei);
    676:   if ((*mapping)->ops->destroy) {
    677:     (*(*mapping)->ops->destroy)(*mapping);
    678:   }
    679:   PetscHeaderDestroy(mapping);
    680:   *mapping = NULL;
    681:   return 0;
    682: }

    684: /*@
    685:     ISLocalToGlobalMappingApplyIS - Creates from an IS in the local numbering
    686:     a new index set using the global numbering defined in an ISLocalToGlobalMapping
    687:     context.

    689:     Collective on is

    691:     Input Parameters:
    692: +   mapping - mapping between local and global numbering
    693: -   is - index set in local numbering

    695:     Output Parameters:
    696: .   newis - index set in global numbering

    698:     Notes:
    699:     The output IS will have the same communicator of the input IS.

    701:     Level: advanced

    703: .seealso: ISLocalToGlobalMappingApply(), ISLocalToGlobalMappingCreate(),
    704:           ISLocalToGlobalMappingDestroy(), ISGlobalToLocalMappingApply()
    705: @*/
    706: PetscErrorCode  ISLocalToGlobalMappingApplyIS(ISLocalToGlobalMapping mapping,IS is,IS *newis)
    707: {
    708:   PetscInt       n,*idxout;
    709:   const PetscInt *idxin;

    715:   ISGetLocalSize(is,&n);
    716:   ISGetIndices(is,&idxin);
    717:   PetscMalloc1(n,&idxout);
    718:   ISLocalToGlobalMappingApply(mapping,n,idxin,idxout);
    719:   ISRestoreIndices(is,&idxin);
    720:   ISCreateGeneral(PetscObjectComm((PetscObject)is),n,idxout,PETSC_OWN_POINTER,newis);
    721:   return 0;
    722: }

    724: /*@
    725:    ISLocalToGlobalMappingApply - Takes a list of integers in a local numbering
    726:    and converts them to the global numbering.

    728:    Not collective

    730:    Input Parameters:
    731: +  mapping - the local to global mapping context
    732: .  N - number of integers
    733: -  in - input indices in local numbering

    735:    Output Parameter:
    736: .  out - indices in global numbering

    738:    Notes:
    739:    The in and out array parameters may be identical.

    741:    Level: advanced

    743: .seealso: ISLocalToGlobalMappingApplyBlock(), ISLocalToGlobalMappingCreate(),ISLocalToGlobalMappingDestroy(),
    744:           ISLocalToGlobalMappingApplyIS(),AOCreateBasic(),AOApplicationToPetsc(),
    745:           AOPetscToApplication(), ISGlobalToLocalMappingApply()

    747: @*/
    748: PetscErrorCode ISLocalToGlobalMappingApply(ISLocalToGlobalMapping mapping,PetscInt N,const PetscInt in[],PetscInt out[])
    749: {
    750:   PetscInt i,bs,Nmax;

    753:   bs   = mapping->bs;
    754:   Nmax = bs*mapping->n;
    755:   if (bs == 1) {
    756:     const PetscInt *idx = mapping->indices;
    757:     for (i=0; i<N; i++) {
    758:       if (in[i] < 0) {
    759:         out[i] = in[i];
    760:         continue;
    761:       }
    763:       out[i] = idx[in[i]];
    764:     }
    765:   } else {
    766:     const PetscInt *idx = mapping->indices;
    767:     for (i=0; i<N; i++) {
    768:       if (in[i] < 0) {
    769:         out[i] = in[i];
    770:         continue;
    771:       }
    773:       out[i] = idx[in[i]/bs]*bs + (in[i] % bs);
    774:     }
    775:   }
    776:   return 0;
    777: }

    779: /*@
    780:    ISLocalToGlobalMappingApplyBlock - Takes a list of integers in a local block numbering and converts them to the global block numbering

    782:    Not collective

    784:    Input Parameters:
    785: +  mapping - the local to global mapping context
    786: .  N - number of integers
    787: -  in - input indices in local block numbering

    789:    Output Parameter:
    790: .  out - indices in global block numbering

    792:    Notes:
    793:    The in and out array parameters may be identical.

    795:    Example:
    796:      If the index values are {0,1,6,7} set with a call to ISLocalToGlobalMappingCreate(PETSC_COMM_SELF,2,2,{0,3}) then the mapping applied to 0
    797:      (the first block) would produce 0 and the mapping applied to 1 (the second block) would produce 3.

    799:    Level: advanced

    801: .seealso: ISLocalToGlobalMappingApply(), ISLocalToGlobalMappingCreate(),ISLocalToGlobalMappingDestroy(),
    802:           ISLocalToGlobalMappingApplyIS(),AOCreateBasic(),AOApplicationToPetsc(),
    803:           AOPetscToApplication(), ISGlobalToLocalMappingApply()

    805: @*/
    806: PetscErrorCode ISLocalToGlobalMappingApplyBlock(ISLocalToGlobalMapping mapping,PetscInt N,const PetscInt in[],PetscInt out[])
    807: {
    808:   PetscInt       i,Nmax;
    809:   const PetscInt *idx;

    812:   Nmax = mapping->n;
    813:   idx = mapping->indices;
    814:   for (i=0; i<N; i++) {
    815:     if (in[i] < 0) {
    816:       out[i] = in[i];
    817:       continue;
    818:     }
    820:     out[i] = idx[in[i]];
    821:   }
    822:   return 0;
    823: }

    825: /*@
    826:     ISGlobalToLocalMappingApply - Provides the local numbering for a list of integers
    827:     specified with a global numbering.

    829:     Not collective

    831:     Input Parameters:
    832: +   mapping - mapping between local and global numbering
    833: .   type - IS_GTOLM_MASK - maps global indices with no local value to -1 in the output list (i.e., mask them)
    834:            IS_GTOLM_DROP - drops the indices with no local value from the output list
    835: .   n - number of global indices to map
    836: -   idx - global indices to map

    838:     Output Parameters:
    839: +   nout - number of indices in output array (if type == IS_GTOLM_MASK then nout = n)
    840: -   idxout - local index of each global index, one must pass in an array long enough
    841:              to hold all the indices. You can call ISGlobalToLocalMappingApply() with
    842:              idxout == NULL to determine the required length (returned in nout)
    843:              and then allocate the required space and call ISGlobalToLocalMappingApply()
    844:              a second time to set the values.

    846:     Notes:
    847:     Either nout or idxout may be NULL. idx and idxout may be identical.

    849:     For "small" problems when using ISGlobalToLocalMappingApply() and ISGlobalToLocalMappingApplyBlock(), the ISLocalToGlobalMappingType of ISLOCALTOGLOBALMAPPINGBASIC will be used;
    850:     this uses more memory but is faster; this approach is not scalable for extremely large mappings. For large problems ISLOCALTOGLOBALMAPPINGHASH is used, this is scalable.
    851:     Use ISLocalToGlobalMappingSetType() or call ISLocalToGlobalMappingSetFromOptions() with the option -islocaltoglobalmapping_type <basic,hash> to control which is used.

    853:     Level: advanced

    855:     Developer Note: The manual page states that idx and idxout may be identical but the calling
    856:        sequence declares idx as const so it cannot be the same as idxout.

    858: .seealso: ISLocalToGlobalMappingApply(), ISGlobalToLocalMappingApplyBlock(), ISLocalToGlobalMappingCreate(),
    859:           ISLocalToGlobalMappingDestroy()
    860: @*/
    861: PetscErrorCode  ISGlobalToLocalMappingApply(ISLocalToGlobalMapping mapping,ISGlobalToLocalMappingMode type,PetscInt n,const PetscInt idx[],PetscInt *nout,PetscInt idxout[])
    862: {
    864:   if (!mapping->data) {
    865:     ISGlobalToLocalMappingSetUp(mapping);
    866:   }
    867:   (*mapping->ops->globaltolocalmappingapply)(mapping,type,n,idx,nout,idxout);
    868:   return 0;
    869: }

    871: /*@
    872:     ISGlobalToLocalMappingApplyIS - Creates from an IS in the global numbering
    873:     a new index set using the local numbering defined in an ISLocalToGlobalMapping
    874:     context.

    876:     Not collective

    878:     Input Parameters:
    879: +   mapping - mapping between local and global numbering
    880: .   type - IS_GTOLM_MASK - maps global indices with no local value to -1 in the output list (i.e., mask them)
    881:            IS_GTOLM_DROP - drops the indices with no local value from the output list
    882: -   is - index set in global numbering

    884:     Output Parameters:
    885: .   newis - index set in local numbering

    887:     Notes:
    888:     The output IS will be sequential, as it encodes a purely local operation

    890:     Level: advanced

    892: .seealso: ISGlobalToLocalMappingApply(), ISLocalToGlobalMappingCreate(),
    893:           ISLocalToGlobalMappingDestroy()
    894: @*/
    895: PetscErrorCode  ISGlobalToLocalMappingApplyIS(ISLocalToGlobalMapping mapping,ISGlobalToLocalMappingMode type,IS is,IS *newis)
    896: {
    897:   PetscInt       n,nout,*idxout;
    898:   const PetscInt *idxin;

    904:   ISGetLocalSize(is,&n);
    905:   ISGetIndices(is,&idxin);
    906:   if (type == IS_GTOLM_MASK) {
    907:     PetscMalloc1(n,&idxout);
    908:   } else {
    909:     ISGlobalToLocalMappingApply(mapping,type,n,idxin,&nout,NULL);
    910:     PetscMalloc1(nout,&idxout);
    911:   }
    912:   ISGlobalToLocalMappingApply(mapping,type,n,idxin,&nout,idxout);
    913:   ISRestoreIndices(is,&idxin);
    914:   ISCreateGeneral(PETSC_COMM_SELF,nout,idxout,PETSC_OWN_POINTER,newis);
    915:   return 0;
    916: }

    918: /*@
    919:     ISGlobalToLocalMappingApplyBlock - Provides the local block numbering for a list of integers
    920:     specified with a block global numbering.

    922:     Not collective

    924:     Input Parameters:
    925: +   mapping - mapping between local and global numbering
    926: .   type - IS_GTOLM_MASK - maps global indices with no local value to -1 in the output list (i.e., mask them)
    927:            IS_GTOLM_DROP - drops the indices with no local value from the output list
    928: .   n - number of global indices to map
    929: -   idx - global indices to map

    931:     Output Parameters:
    932: +   nout - number of indices in output array (if type == IS_GTOLM_MASK then nout = n)
    933: -   idxout - local index of each global index, one must pass in an array long enough
    934:              to hold all the indices. You can call ISGlobalToLocalMappingApplyBlock() with
    935:              idxout == NULL to determine the required length (returned in nout)
    936:              and then allocate the required space and call ISGlobalToLocalMappingApplyBlock()
    937:              a second time to set the values.

    939:     Notes:
    940:     Either nout or idxout may be NULL. idx and idxout may be identical.

    942:     For "small" problems when using ISGlobalToLocalMappingApply() and ISGlobalToLocalMappingApplyBlock(), the ISLocalToGlobalMappingType of ISLOCALTOGLOBALMAPPINGBASIC will be used;
    943:     this uses more memory but is faster; this approach is not scalable for extremely large mappings. For large problems ISLOCALTOGLOBALMAPPINGHASH is used, this is scalable.
    944:     Use ISLocalToGlobalMappingSetType() or call ISLocalToGlobalMappingSetFromOptions() with the option -islocaltoglobalmapping_type <basic,hash> to control which is used.

    946:     Level: advanced

    948:     Developer Note: The manual page states that idx and idxout may be identical but the calling
    949:        sequence declares idx as const so it cannot be the same as idxout.

    951: .seealso: ISLocalToGlobalMappingApply(), ISGlobalToLocalMappingApply(), ISLocalToGlobalMappingCreate(),
    952:           ISLocalToGlobalMappingDestroy()
    953: @*/
    954: PetscErrorCode  ISGlobalToLocalMappingApplyBlock(ISLocalToGlobalMapping mapping,ISGlobalToLocalMappingMode type,
    955:                                                  PetscInt n,const PetscInt idx[],PetscInt *nout,PetscInt idxout[])
    956: {
    958:   if (!mapping->data) {
    959:     ISGlobalToLocalMappingSetUp(mapping);
    960:   }
    961:   (*mapping->ops->globaltolocalmappingapplyblock)(mapping,type,n,idx,nout,idxout);
    962:   return 0;
    963: }

    965: /*@C
    966:     ISLocalToGlobalMappingGetBlockInfo - Gets the neighbor information for each processor and
    967:      each index shared by more than one processor

    969:     Collective on ISLocalToGlobalMapping

    971:     Input Parameter:
    972: .   mapping - the mapping from local to global indexing

    974:     Output Parameters:
    975: +   nproc - number of processors that are connected to this one
    976: .   proc - neighboring processors
    977: .   numproc - number of indices for each subdomain (processor)
    978: -   indices - indices of nodes (in local numbering) shared with neighbors (sorted by global numbering)

    980:     Level: advanced

    982:     Fortran Usage:
    983: $        ISLocalToGlobalMpngGetInfoSize(ISLocalToGlobalMapping,PetscInt nproc,PetscInt numprocmax,ierr) followed by
    984: $        ISLocalToGlobalMappingGetInfo(ISLocalToGlobalMapping,PetscInt nproc, PetscInt procs[nproc],PetscInt numprocs[nproc],
    985:           PetscInt indices[nproc][numprocmax],ierr)
    986:         There is no ISLocalToGlobalMappingRestoreInfo() in Fortran. You must make sure that procs[], numprocs[] and
    987:         indices[][] are large enough arrays, either by allocating them dynamically or defining static ones large enough.

    989: .seealso: ISLocalToGlobalMappingDestroy(), ISLocalToGlobalMappingCreateIS(), ISLocalToGlobalMappingCreate(),
    990:           ISLocalToGlobalMappingRestoreInfo()
    991: @*/
    992: PetscErrorCode  ISLocalToGlobalMappingGetBlockInfo(ISLocalToGlobalMapping mapping,PetscInt *nproc,PetscInt *procs[],PetscInt *numprocs[],PetscInt **indices[])
    993: {
    995:   if (mapping->info_cached) {
    996:     *nproc    = mapping->info_nproc;
    997:     *procs    = mapping->info_procs;
    998:     *numprocs = mapping->info_numprocs;
    999:     *indices  = mapping->info_indices;
    1000:   } else {
    1001:     ISLocalToGlobalMappingGetBlockInfo_Private(mapping,nproc,procs,numprocs,indices);
    1002:   }
    1003:   return 0;
    1004: }

    1006: static PetscErrorCode  ISLocalToGlobalMappingGetBlockInfo_Private(ISLocalToGlobalMapping mapping,PetscInt *nproc,PetscInt *procs[],PetscInt *numprocs[],PetscInt **indices[])
    1007: {
    1008:   PetscMPIInt    size,rank,tag1,tag2,tag3,*len,*source,imdex;
    1009:   PetscInt       i,n = mapping->n,Ng,ng,max = 0,*lindices = mapping->indices;
    1010:   PetscInt       *nprocs,*owner,nsends,*sends,j,*starts,nmax,nrecvs,*recvs,proc;
    1011:   PetscInt       cnt,scale,*ownedsenders,*nownedsenders,rstart,nowned;
    1012:   PetscInt       node,nownedm,nt,*sends2,nsends2,*starts2,*lens2,*dest,nrecvs2,*starts3,*recvs2,k,*bprocs,*tmp;
    1013:   PetscInt       first_procs,first_numprocs,*first_indices;
    1014:   MPI_Request    *recv_waits,*send_waits;
    1015:   MPI_Status     recv_status,*send_status,*recv_statuses;
    1016:   MPI_Comm       comm;
    1017:   PetscBool      debug = PETSC_FALSE;

    1019:   PetscObjectGetComm((PetscObject)mapping,&comm);
    1020:   MPI_Comm_size(comm,&size);
    1021:   MPI_Comm_rank(comm,&rank);
    1022:   if (size == 1) {
    1023:     *nproc         = 0;
    1024:     *procs         = NULL;
    1025:     PetscNew(numprocs);
    1026:     (*numprocs)[0] = 0;
    1027:     PetscNew(indices);
    1028:     (*indices)[0]  = NULL;
    1029:     /* save info for reuse */
    1030:     mapping->info_nproc = *nproc;
    1031:     mapping->info_procs = *procs;
    1032:     mapping->info_numprocs = *numprocs;
    1033:     mapping->info_indices = *indices;
    1034:     mapping->info_cached = PETSC_TRUE;
    1035:     return 0;
    1036:   }

    1038:   PetscOptionsGetBool(((PetscObject)mapping)->options,NULL,"-islocaltoglobalmappinggetinfo_debug",&debug,NULL);

    1040:   /*
    1041:     Notes on ISLocalToGlobalMappingGetBlockInfo

    1043:     globally owned node - the nodes that have been assigned to this processor in global
    1044:            numbering, just for this routine.

    1046:     nontrivial globally owned node - node assigned to this processor that is on a subdomain
    1047:            boundary (i.e. is has more than one local owner)

    1049:     locally owned node - node that exists on this processors subdomain

    1051:     nontrivial locally owned node - node that is not in the interior (i.e. has more than one
    1052:            local subdomain
    1053:   */
    1054:   PetscObjectGetNewTag((PetscObject)mapping,&tag1);
    1055:   PetscObjectGetNewTag((PetscObject)mapping,&tag2);
    1056:   PetscObjectGetNewTag((PetscObject)mapping,&tag3);

    1058:   for (i=0; i<n; i++) {
    1059:     if (lindices[i] > max) max = lindices[i];
    1060:   }
    1061:   MPIU_Allreduce(&max,&Ng,1,MPIU_INT,MPI_MAX,comm);
    1062:   Ng++;
    1063:   MPI_Comm_size(comm,&size);
    1064:   MPI_Comm_rank(comm,&rank);
    1065:   scale  = Ng/size + 1;
    1066:   ng     = scale; if (rank == size-1) ng = Ng - scale*(size-1); ng = PetscMax(1,ng);
    1067:   rstart = scale*rank;

    1069:   /* determine ownership ranges of global indices */
    1070:   PetscMalloc1(2*size,&nprocs);
    1071:   PetscArrayzero(nprocs,2*size);

    1073:   /* determine owners of each local node  */
    1074:   PetscMalloc1(n,&owner);
    1075:   for (i=0; i<n; i++) {
    1076:     proc             = lindices[i]/scale; /* processor that globally owns this index */
    1077:     nprocs[2*proc+1] = 1;                 /* processor globally owns at least one of ours */
    1078:     owner[i]         = proc;
    1079:     nprocs[2*proc]++;                     /* count of how many that processor globally owns of ours */
    1080:   }
    1081:   nsends = 0; for (i=0; i<size; i++) nsends += nprocs[2*i+1];
    1082:   PetscInfo(mapping,"Number of global owners for my local data %" PetscInt_FMT "\n",nsends);

    1084:   /* inform other processors of number of messages and max length*/
    1085:   PetscMaxSum(comm,nprocs,&nmax,&nrecvs);
    1086:   PetscInfo(mapping,"Number of local owners for my global data %" PetscInt_FMT "\n",nrecvs);

    1088:   /* post receives for owned rows */
    1089:   PetscMalloc1((2*nrecvs+1)*(nmax+1),&recvs);
    1090:   PetscMalloc1(nrecvs+1,&recv_waits);
    1091:   for (i=0; i<nrecvs; i++) {
    1092:     MPI_Irecv(recvs+2*nmax*i,2*nmax,MPIU_INT,MPI_ANY_SOURCE,tag1,comm,recv_waits+i);
    1093:   }

    1095:   /* pack messages containing lists of local nodes to owners */
    1096:   PetscMalloc1(2*n+1,&sends);
    1097:   PetscMalloc1(size+1,&starts);
    1098:   starts[0] = 0;
    1099:   for (i=1; i<size; i++) starts[i] = starts[i-1] + 2*nprocs[2*i-2];
    1100:   for (i=0; i<n; i++) {
    1101:     sends[starts[owner[i]]++] = lindices[i];
    1102:     sends[starts[owner[i]]++] = i;
    1103:   }
    1104:   PetscFree(owner);
    1105:   starts[0] = 0;
    1106:   for (i=1; i<size; i++) starts[i] = starts[i-1] + 2*nprocs[2*i-2];

    1108:   /* send the messages */
    1109:   PetscMalloc1(nsends+1,&send_waits);
    1110:   PetscMalloc1(nsends+1,&dest);
    1111:   cnt = 0;
    1112:   for (i=0; i<size; i++) {
    1113:     if (nprocs[2*i]) {
    1114:       MPI_Isend(sends+starts[i],2*nprocs[2*i],MPIU_INT,i,tag1,comm,send_waits+cnt);
    1115:       dest[cnt] = i;
    1116:       cnt++;
    1117:     }
    1118:   }
    1119:   PetscFree(starts);

    1121:   /* wait on receives */
    1122:   PetscMalloc1(nrecvs+1,&source);
    1123:   PetscMalloc1(nrecvs+1,&len);
    1124:   cnt  = nrecvs;
    1125:   PetscCalloc1(ng+1,&nownedsenders);
    1126:   while (cnt) {
    1127:     MPI_Waitany(nrecvs,recv_waits,&imdex,&recv_status);
    1128:     /* unpack receives into our local space */
    1129:     MPI_Get_count(&recv_status,MPIU_INT,&len[imdex]);
    1130:     source[imdex] = recv_status.MPI_SOURCE;
    1131:     len[imdex]    = len[imdex]/2;
    1132:     /* count how many local owners for each of my global owned indices */
    1133:     for (i=0; i<len[imdex]; i++) nownedsenders[recvs[2*imdex*nmax+2*i]-rstart]++;
    1134:     cnt--;
    1135:   }
    1136:   PetscFree(recv_waits);

    1138:   /* count how many globally owned indices are on an edge multiplied by how many processors own them. */
    1139:   nowned  = 0;
    1140:   nownedm = 0;
    1141:   for (i=0; i<ng; i++) {
    1142:     if (nownedsenders[i] > 1) {nownedm += nownedsenders[i]; nowned++;}
    1143:   }

    1145:   /* create single array to contain rank of all local owners of each globally owned index */
    1146:   PetscMalloc1(nownedm+1,&ownedsenders);
    1147:   PetscMalloc1(ng+1,&starts);
    1148:   starts[0] = 0;
    1149:   for (i=1; i<ng; i++) {
    1150:     if (nownedsenders[i-1] > 1) starts[i] = starts[i-1] + nownedsenders[i-1];
    1151:     else starts[i] = starts[i-1];
    1152:   }

    1154:   /* for each nontrival globally owned node list all arriving processors */
    1155:   for (i=0; i<nrecvs; i++) {
    1156:     for (j=0; j<len[i]; j++) {
    1157:       node = recvs[2*i*nmax+2*j]-rstart;
    1158:       if (nownedsenders[node] > 1) ownedsenders[starts[node]++] = source[i];
    1159:     }
    1160:   }

    1162:   if (debug) { /* -----------------------------------  */
    1163:     starts[0] = 0;
    1164:     for (i=1; i<ng; i++) {
    1165:       if (nownedsenders[i-1] > 1) starts[i] = starts[i-1] + nownedsenders[i-1];
    1166:       else starts[i] = starts[i-1];
    1167:     }
    1168:     for (i=0; i<ng; i++) {
    1169:       if (nownedsenders[i] > 1) {
    1170:         PetscSynchronizedPrintf(comm,"[%d] global node %" PetscInt_FMT " local owner processors: ",rank,i+rstart);
    1171:         for (j=0; j<nownedsenders[i]; j++) {
    1172:           PetscSynchronizedPrintf(comm,"%" PetscInt_FMT " ",ownedsenders[starts[i]+j]);
    1173:         }
    1174:         PetscSynchronizedPrintf(comm,"\n");
    1175:       }
    1176:     }
    1177:     PetscSynchronizedFlush(comm,PETSC_STDOUT);
    1178:   } /* -----------------------------------  */

    1180:   /* wait on original sends */
    1181:   if (nsends) {
    1182:     PetscMalloc1(nsends,&send_status);
    1183:     MPI_Waitall(nsends,send_waits,send_status);
    1184:     PetscFree(send_status);
    1185:   }
    1186:   PetscFree(send_waits);
    1187:   PetscFree(sends);
    1188:   PetscFree(nprocs);

    1190:   /* pack messages to send back to local owners */
    1191:   starts[0] = 0;
    1192:   for (i=1; i<ng; i++) {
    1193:     if (nownedsenders[i-1] > 1) starts[i] = starts[i-1] + nownedsenders[i-1];
    1194:     else starts[i] = starts[i-1];
    1195:   }
    1196:   nsends2 = nrecvs;
    1197:   PetscMalloc1(nsends2+1,&nprocs); /* length of each message */
    1198:   for (i=0; i<nrecvs; i++) {
    1199:     nprocs[i] = 1;
    1200:     for (j=0; j<len[i]; j++) {
    1201:       node = recvs[2*i*nmax+2*j]-rstart;
    1202:       if (nownedsenders[node] > 1) nprocs[i] += 2 + nownedsenders[node];
    1203:     }
    1204:   }
    1205:   nt = 0;
    1206:   for (i=0; i<nsends2; i++) nt += nprocs[i];

    1208:   PetscMalloc1(nt+1,&sends2);
    1209:   PetscMalloc1(nsends2+1,&starts2);

    1211:   starts2[0] = 0;
    1212:   for (i=1; i<nsends2; i++) starts2[i] = starts2[i-1] + nprocs[i-1];
    1213:   /*
    1214:      Each message is 1 + nprocs[i] long, and consists of
    1215:        (0) the number of nodes being sent back
    1216:        (1) the local node number,
    1217:        (2) the number of processors sharing it,
    1218:        (3) the processors sharing it
    1219:   */
    1220:   for (i=0; i<nsends2; i++) {
    1221:     cnt = 1;
    1222:     sends2[starts2[i]] = 0;
    1223:     for (j=0; j<len[i]; j++) {
    1224:       node = recvs[2*i*nmax+2*j]-rstart;
    1225:       if (nownedsenders[node] > 1) {
    1226:         sends2[starts2[i]]++;
    1227:         sends2[starts2[i]+cnt++] = recvs[2*i*nmax+2*j+1];
    1228:         sends2[starts2[i]+cnt++] = nownedsenders[node];
    1229:         PetscArraycpy(&sends2[starts2[i]+cnt],&ownedsenders[starts[node]],nownedsenders[node]);
    1230:         cnt += nownedsenders[node];
    1231:       }
    1232:     }
    1233:   }

    1235:   /* receive the message lengths */
    1236:   nrecvs2 = nsends;
    1237:   PetscMalloc1(nrecvs2+1,&lens2);
    1238:   PetscMalloc1(nrecvs2+1,&starts3);
    1239:   PetscMalloc1(nrecvs2+1,&recv_waits);
    1240:   for (i=0; i<nrecvs2; i++) {
    1241:     MPI_Irecv(&lens2[i],1,MPIU_INT,dest[i],tag2,comm,recv_waits+i);
    1242:   }

    1244:   /* send the message lengths */
    1245:   for (i=0; i<nsends2; i++) {
    1246:     MPI_Send(&nprocs[i],1,MPIU_INT,source[i],tag2,comm);
    1247:   }

    1249:   /* wait on receives of lens */
    1250:   if (nrecvs2) {
    1251:     PetscMalloc1(nrecvs2,&recv_statuses);
    1252:     MPI_Waitall(nrecvs2,recv_waits,recv_statuses);
    1253:     PetscFree(recv_statuses);
    1254:   }
    1255:   PetscFree(recv_waits);

    1257:   starts3[0] = 0;
    1258:   nt         = 0;
    1259:   for (i=0; i<nrecvs2-1; i++) {
    1260:     starts3[i+1] = starts3[i] + lens2[i];
    1261:     nt          += lens2[i];
    1262:   }
    1263:   if (nrecvs2) nt += lens2[nrecvs2-1];

    1265:   PetscMalloc1(nt+1,&recvs2);
    1266:   PetscMalloc1(nrecvs2+1,&recv_waits);
    1267:   for (i=0; i<nrecvs2; i++) {
    1268:     MPI_Irecv(recvs2+starts3[i],lens2[i],MPIU_INT,dest[i],tag3,comm,recv_waits+i);
    1269:   }

    1271:   /* send the messages */
    1272:   PetscMalloc1(nsends2+1,&send_waits);
    1273:   for (i=0; i<nsends2; i++) {
    1274:     MPI_Isend(sends2+starts2[i],nprocs[i],MPIU_INT,source[i],tag3,comm,send_waits+i);
    1275:   }

    1277:   /* wait on receives */
    1278:   if (nrecvs2) {
    1279:     PetscMalloc1(nrecvs2,&recv_statuses);
    1280:     MPI_Waitall(nrecvs2,recv_waits,recv_statuses);
    1281:     PetscFree(recv_statuses);
    1282:   }
    1283:   PetscFree(recv_waits);
    1284:   PetscFree(nprocs);

    1286:   if (debug) { /* -----------------------------------  */
    1287:     cnt = 0;
    1288:     for (i=0; i<nrecvs2; i++) {
    1289:       nt = recvs2[cnt++];
    1290:       for (j=0; j<nt; j++) {
    1291:         PetscSynchronizedPrintf(comm,"[%d] local node %" PetscInt_FMT " number of subdomains %" PetscInt_FMT ": ",rank,recvs2[cnt],recvs2[cnt+1]);
    1292:         for (k=0; k<recvs2[cnt+1]; k++) {
    1293:           PetscSynchronizedPrintf(comm,"%" PetscInt_FMT " ",recvs2[cnt+2+k]);
    1294:         }
    1295:         cnt += 2 + recvs2[cnt+1];
    1296:         PetscSynchronizedPrintf(comm,"\n");
    1297:       }
    1298:     }
    1299:     PetscSynchronizedFlush(comm,PETSC_STDOUT);
    1300:   } /* -----------------------------------  */

    1302:   /* count number subdomains for each local node */
    1303:   PetscCalloc1(size,&nprocs);
    1304:   cnt  = 0;
    1305:   for (i=0; i<nrecvs2; i++) {
    1306:     nt = recvs2[cnt++];
    1307:     for (j=0; j<nt; j++) {
    1308:       for (k=0; k<recvs2[cnt+1]; k++) nprocs[recvs2[cnt+2+k]]++;
    1309:       cnt += 2 + recvs2[cnt+1];
    1310:     }
    1311:   }
    1312:   nt = 0; for (i=0; i<size; i++) nt += (nprocs[i] > 0);
    1313:   *nproc    = nt;
    1314:   PetscMalloc1(nt+1,procs);
    1315:   PetscMalloc1(nt+1,numprocs);
    1316:   PetscMalloc1(nt+1,indices);
    1317:   for (i=0;i<nt+1;i++) (*indices)[i]=NULL;
    1318:   PetscMalloc1(size,&bprocs);
    1319:   cnt  = 0;
    1320:   for (i=0; i<size; i++) {
    1321:     if (nprocs[i] > 0) {
    1322:       bprocs[i]        = cnt;
    1323:       (*procs)[cnt]    = i;
    1324:       (*numprocs)[cnt] = nprocs[i];
    1325:       PetscMalloc1(nprocs[i],&(*indices)[cnt]);
    1326:       cnt++;
    1327:     }
    1328:   }

    1330:   /* make the list of subdomains for each nontrivial local node */
    1331:   PetscArrayzero(*numprocs,nt);
    1332:   cnt  = 0;
    1333:   for (i=0; i<nrecvs2; i++) {
    1334:     nt = recvs2[cnt++];
    1335:     for (j=0; j<nt; j++) {
    1336:       for (k=0; k<recvs2[cnt+1]; k++) (*indices)[bprocs[recvs2[cnt+2+k]]][(*numprocs)[bprocs[recvs2[cnt+2+k]]]++] = recvs2[cnt];
    1337:       cnt += 2 + recvs2[cnt+1];
    1338:     }
    1339:   }
    1340:   PetscFree(bprocs);
    1341:   PetscFree(recvs2);

    1343:   /* sort the node indexing by their global numbers */
    1344:   nt = *nproc;
    1345:   for (i=0; i<nt; i++) {
    1346:     PetscMalloc1((*numprocs)[i],&tmp);
    1347:     for (j=0; j<(*numprocs)[i]; j++) tmp[j] = lindices[(*indices)[i][j]];
    1348:     PetscSortIntWithArray((*numprocs)[i],tmp,(*indices)[i]);
    1349:     PetscFree(tmp);
    1350:   }

    1352:   if (debug) { /* -----------------------------------  */
    1353:     nt = *nproc;
    1354:     for (i=0; i<nt; i++) {
    1355:       PetscSynchronizedPrintf(comm,"[%d] subdomain %" PetscInt_FMT " number of indices %" PetscInt_FMT ": ",rank,(*procs)[i],(*numprocs)[i]);
    1356:       for (j=0; j<(*numprocs)[i]; j++) {
    1357:         PetscSynchronizedPrintf(comm,"%" PetscInt_FMT " ",(*indices)[i][j]);
    1358:       }
    1359:       PetscSynchronizedPrintf(comm,"\n");
    1360:     }
    1361:     PetscSynchronizedFlush(comm,PETSC_STDOUT);
    1362:   } /* -----------------------------------  */

    1364:   /* wait on sends */
    1365:   if (nsends2) {
    1366:     PetscMalloc1(nsends2,&send_status);
    1367:     MPI_Waitall(nsends2,send_waits,send_status);
    1368:     PetscFree(send_status);
    1369:   }

    1371:   PetscFree(starts3);
    1372:   PetscFree(dest);
    1373:   PetscFree(send_waits);

    1375:   PetscFree(nownedsenders);
    1376:   PetscFree(ownedsenders);
    1377:   PetscFree(starts);
    1378:   PetscFree(starts2);
    1379:   PetscFree(lens2);

    1381:   PetscFree(source);
    1382:   PetscFree(len);
    1383:   PetscFree(recvs);
    1384:   PetscFree(nprocs);
    1385:   PetscFree(sends2);

    1387:   /* put the information about myself as the first entry in the list */
    1388:   first_procs    = (*procs)[0];
    1389:   first_numprocs = (*numprocs)[0];
    1390:   first_indices  = (*indices)[0];
    1391:   for (i=0; i<*nproc; i++) {
    1392:     if ((*procs)[i] == rank) {
    1393:       (*procs)[0]    = (*procs)[i];
    1394:       (*numprocs)[0] = (*numprocs)[i];
    1395:       (*indices)[0]  = (*indices)[i];
    1396:       (*procs)[i]    = first_procs;
    1397:       (*numprocs)[i] = first_numprocs;
    1398:       (*indices)[i]  = first_indices;
    1399:       break;
    1400:     }
    1401:   }

    1403:   /* save info for reuse */
    1404:   mapping->info_nproc = *nproc;
    1405:   mapping->info_procs = *procs;
    1406:   mapping->info_numprocs = *numprocs;
    1407:   mapping->info_indices = *indices;
    1408:   mapping->info_cached = PETSC_TRUE;
    1409:   return 0;
    1410: }

    1412: /*@C
    1413:     ISLocalToGlobalMappingRestoreBlockInfo - Frees the memory allocated by ISLocalToGlobalMappingGetBlockInfo()

    1415:     Collective on ISLocalToGlobalMapping

    1417:     Input Parameter:
    1418: .   mapping - the mapping from local to global indexing

    1420:     Output Parameters:
    1421: +   nproc - number of processors that are connected to this one
    1422: .   proc - neighboring processors
    1423: .   numproc - number of indices for each processor
    1424: -   indices - indices of local nodes shared with neighbor (sorted by global numbering)

    1426:     Level: advanced

    1428: .seealso: ISLocalToGlobalMappingDestroy(), ISLocalToGlobalMappingCreateIS(), ISLocalToGlobalMappingCreate(),
    1429:           ISLocalToGlobalMappingGetInfo()
    1430: @*/
    1431: PetscErrorCode  ISLocalToGlobalMappingRestoreBlockInfo(ISLocalToGlobalMapping mapping,PetscInt *nproc,PetscInt *procs[],PetscInt *numprocs[],PetscInt **indices[])
    1432: {
    1434:   if (mapping->info_free) {
    1435:     PetscFree(*numprocs);
    1436:     if (*indices) {
    1437:       PetscInt i;

    1439:       PetscFree((*indices)[0]);
    1440:       for (i=1; i<*nproc; i++) {
    1441:         PetscFree((*indices)[i]);
    1442:       }
    1443:       PetscFree(*indices);
    1444:     }
    1445:   }
    1446:   *nproc    = 0;
    1447:   *procs    = NULL;
    1448:   *numprocs = NULL;
    1449:   *indices  = NULL;
    1450:   return 0;
    1451: }

    1453: /*@C
    1454:     ISLocalToGlobalMappingGetInfo - Gets the neighbor information for each processor and
    1455:      each index shared by more than one processor

    1457:     Collective on ISLocalToGlobalMapping

    1459:     Input Parameter:
    1460: .   mapping - the mapping from local to global indexing

    1462:     Output Parameters:
    1463: +   nproc - number of processors that are connected to this one
    1464: .   proc - neighboring processors
    1465: .   numproc - number of indices for each subdomain (processor)
    1466: -   indices - indices of nodes (in local numbering) shared with neighbors (sorted by global numbering)

    1468:     Level: advanced

    1470:     Notes: The user needs to call ISLocalToGlobalMappingRestoreInfo when the data is no longer needed.

    1472:     Fortran Usage:
    1473: $        ISLocalToGlobalMpngGetInfoSize(ISLocalToGlobalMapping,PetscInt nproc,PetscInt numprocmax,ierr) followed by
    1474: $        ISLocalToGlobalMappingGetInfo(ISLocalToGlobalMapping,PetscInt nproc, PetscInt procs[nproc],PetscInt numprocs[nproc],
    1475:           PetscInt indices[nproc][numprocmax],ierr)
    1476:         There is no ISLocalToGlobalMappingRestoreInfo() in Fortran. You must make sure that procs[], numprocs[] and
    1477:         indices[][] are large enough arrays, either by allocating them dynamically or defining static ones large enough.

    1479: .seealso: ISLocalToGlobalMappingDestroy(), ISLocalToGlobalMappingCreateIS(), ISLocalToGlobalMappingCreate(),
    1480:           ISLocalToGlobalMappingRestoreInfo()
    1481: @*/
    1482: PetscErrorCode  ISLocalToGlobalMappingGetInfo(ISLocalToGlobalMapping mapping,PetscInt *nproc,PetscInt *procs[],PetscInt *numprocs[],PetscInt **indices[])
    1483: {
    1484:   PetscInt       **bindices = NULL,*bnumprocs = NULL,bs,i,j,k;

    1487:   bs = mapping->bs;
    1488:   ISLocalToGlobalMappingGetBlockInfo(mapping,nproc,procs,&bnumprocs,&bindices);
    1489:   if (bs > 1) { /* we need to expand the cached info */
    1490:     PetscCalloc1(*nproc,&*indices);
    1491:     PetscCalloc1(*nproc,&*numprocs);
    1492:     for (i=0; i<*nproc; i++) {
    1493:       PetscMalloc1(bs*bnumprocs[i],&(*indices)[i]);
    1494:       for (j=0; j<bnumprocs[i]; j++) {
    1495:         for (k=0; k<bs; k++) {
    1496:           (*indices)[i][j*bs+k] = bs*bindices[i][j] + k;
    1497:         }
    1498:       }
    1499:       (*numprocs)[i] = bnumprocs[i]*bs;
    1500:     }
    1501:     mapping->info_free = PETSC_TRUE;
    1502:   } else {
    1503:     *numprocs = bnumprocs;
    1504:     *indices  = bindices;
    1505:   }
    1506:   return 0;
    1507: }

    1509: /*@C
    1510:     ISLocalToGlobalMappingRestoreInfo - Frees the memory allocated by ISLocalToGlobalMappingGetInfo()

    1512:     Collective on ISLocalToGlobalMapping

    1514:     Input Parameter:
    1515: .   mapping - the mapping from local to global indexing

    1517:     Output Parameters:
    1518: +   nproc - number of processors that are connected to this one
    1519: .   proc - neighboring processors
    1520: .   numproc - number of indices for each processor
    1521: -   indices - indices of local nodes shared with neighbor (sorted by global numbering)

    1523:     Level: advanced

    1525: .seealso: ISLocalToGlobalMappingDestroy(), ISLocalToGlobalMappingCreateIS(), ISLocalToGlobalMappingCreate(),
    1526:           ISLocalToGlobalMappingGetInfo()
    1527: @*/
    1528: PetscErrorCode  ISLocalToGlobalMappingRestoreInfo(ISLocalToGlobalMapping mapping,PetscInt *nproc,PetscInt *procs[],PetscInt *numprocs[],PetscInt **indices[])
    1529: {
    1530:   ISLocalToGlobalMappingRestoreBlockInfo(mapping,nproc,procs,numprocs,indices);
    1531:   return 0;
    1532: }

    1534: /*@C
    1535:     ISLocalToGlobalMappingGetNodeInfo - Gets the neighbor information for each node

    1537:     Collective on ISLocalToGlobalMapping

    1539:     Input Parameter:
    1540: .   mapping - the mapping from local to global indexing

    1542:     Output Parameters:
    1543: +   nnodes - number of local nodes (same ISLocalToGlobalMappingGetSize())
    1544: .   count - number of neighboring processors per node
    1545: -   indices - indices of processes sharing the node (sorted)

    1547:     Level: advanced

    1549:     Notes: The user needs to call ISLocalToGlobalMappingRestoreInfo when the data is no longer needed.

    1551: .seealso: ISLocalToGlobalMappingDestroy(), ISLocalToGlobalMappingCreateIS(), ISLocalToGlobalMappingCreate(),
    1552:           ISLocalToGlobalMappingGetInfo(), ISLocalToGlobalMappingRestoreNodeInfo()
    1553: @*/
    1554: PetscErrorCode  ISLocalToGlobalMappingGetNodeInfo(ISLocalToGlobalMapping mapping,PetscInt *nnodes,PetscInt *count[],PetscInt **indices[])
    1555: {
    1556:   PetscInt       n;

    1559:   ISLocalToGlobalMappingGetSize(mapping,&n);
    1560:   if (!mapping->info_nodec) {
    1561:     PetscInt i,m,n_neigh,*neigh,*n_shared,**shared;

    1563:     PetscMalloc2(n+1,&mapping->info_nodec,n,&mapping->info_nodei);
    1564:     ISLocalToGlobalMappingGetInfo(mapping,&n_neigh,&neigh,&n_shared,&shared);
    1565:     for (i=0;i<n;i++) { mapping->info_nodec[i] = 1;}
    1566:     m = n;
    1567:     mapping->info_nodec[n] = 0;
    1568:     for (i=1;i<n_neigh;i++) {
    1569:       PetscInt j;

    1571:       m += n_shared[i];
    1572:       for (j=0;j<n_shared[i];j++) mapping->info_nodec[shared[i][j]] += 1;
    1573:     }
    1574:     if (n) PetscMalloc1(m,&mapping->info_nodei[0]);
    1575:     for (i=1;i<n;i++) mapping->info_nodei[i] = mapping->info_nodei[i-1] + mapping->info_nodec[i-1];
    1576:     PetscArrayzero(mapping->info_nodec,n);
    1577:     for (i=0;i<n;i++) { mapping->info_nodec[i] = 1; mapping->info_nodei[i][0] = neigh[0]; }
    1578:     for (i=1;i<n_neigh;i++) {
    1579:       PetscInt j;

    1581:       for (j=0;j<n_shared[i];j++) {
    1582:         PetscInt k = shared[i][j];

    1584:         mapping->info_nodei[k][mapping->info_nodec[k]] = neigh[i];
    1585:         mapping->info_nodec[k] += 1;
    1586:       }
    1587:     }
    1588:     for (i=0;i<n;i++) PetscSortRemoveDupsInt(&mapping->info_nodec[i],mapping->info_nodei[i]);
    1589:     ISLocalToGlobalMappingRestoreInfo(mapping,&n_neigh,&neigh,&n_shared,&shared);
    1590:   }
    1591:   if (nnodes)  *nnodes  = n;
    1592:   if (count)   *count   = mapping->info_nodec;
    1593:   if (indices) *indices = mapping->info_nodei;
    1594:   return 0;
    1595: }

    1597: /*@C
    1598:     ISLocalToGlobalMappingRestoreNodeInfo - Frees the memory allocated by ISLocalToGlobalMappingGetNodeInfo()

    1600:     Collective on ISLocalToGlobalMapping

    1602:     Input Parameter:
    1603: .   mapping - the mapping from local to global indexing

    1605:     Output Parameters:
    1606: +   nnodes - number of local nodes
    1607: .   count - number of neighboring processors per node
    1608: -   indices - indices of processes sharing the node (sorted)

    1610:     Level: advanced

    1612: .seealso: ISLocalToGlobalMappingDestroy(), ISLocalToGlobalMappingCreateIS(), ISLocalToGlobalMappingCreate(),
    1613:           ISLocalToGlobalMappingGetInfo()
    1614: @*/
    1615: PetscErrorCode  ISLocalToGlobalMappingRestoreNodeInfo(ISLocalToGlobalMapping mapping,PetscInt *nnodes,PetscInt *count[],PetscInt **indices[])
    1616: {
    1618:   if (nnodes)  *nnodes  = 0;
    1619:   if (count)   *count   = NULL;
    1620:   if (indices) *indices = NULL;
    1621:   return 0;
    1622: }

    1624: /*@C
    1625:    ISLocalToGlobalMappingGetIndices - Get global indices for every local point that is mapped

    1627:    Not Collective

    1629:    Input Parameter:
    1630: . ltog - local to global mapping

    1632:    Output Parameter:
    1633: . array - array of indices, the length of this array may be obtained with ISLocalToGlobalMappingGetSize()

    1635:    Level: advanced

    1637:    Notes:
    1638:     ISLocalToGlobalMappingGetSize() returns the length the this array

    1640: .seealso: ISLocalToGlobalMappingCreate(), ISLocalToGlobalMappingApply(), ISLocalToGlobalMappingRestoreIndices(), ISLocalToGlobalMappingGetBlockIndices(), ISLocalToGlobalMappingRestoreBlockIndices()
    1641: @*/
    1642: PetscErrorCode  ISLocalToGlobalMappingGetIndices(ISLocalToGlobalMapping ltog,const PetscInt **array)
    1643: {
    1646:   if (ltog->bs == 1) {
    1647:     *array = ltog->indices;
    1648:   } else {
    1649:     PetscInt       *jj,k,i,j,n = ltog->n, bs = ltog->bs;
    1650:     const PetscInt *ii;

    1652:     PetscMalloc1(bs*n,&jj);
    1653:     *array = jj;
    1654:     k    = 0;
    1655:     ii   = ltog->indices;
    1656:     for (i=0; i<n; i++)
    1657:       for (j=0; j<bs; j++)
    1658:         jj[k++] = bs*ii[i] + j;
    1659:   }
    1660:   return 0;
    1661: }

    1663: /*@C
    1664:    ISLocalToGlobalMappingRestoreIndices - Restore indices obtained with ISLocalToGlobalMappingGetIndices()

    1666:    Not Collective

    1668:    Input Parameters:
    1669: + ltog - local to global mapping
    1670: - array - array of indices

    1672:    Level: advanced

    1674: .seealso: ISLocalToGlobalMappingCreate(), ISLocalToGlobalMappingApply(), ISLocalToGlobalMappingGetIndices()
    1675: @*/
    1676: PetscErrorCode  ISLocalToGlobalMappingRestoreIndices(ISLocalToGlobalMapping ltog,const PetscInt **array)
    1677: {

    1682:   if (ltog->bs > 1) {
    1683:     PetscFree(*(void**)array);
    1684:   }
    1685:   return 0;
    1686: }

    1688: /*@C
    1689:    ISLocalToGlobalMappingGetBlockIndices - Get global indices for every local block

    1691:    Not Collective

    1693:    Input Parameter:
    1694: . ltog - local to global mapping

    1696:    Output Parameter:
    1697: . array - array of indices

    1699:    Level: advanced

    1701: .seealso: ISLocalToGlobalMappingCreate(), ISLocalToGlobalMappingApply(), ISLocalToGlobalMappingRestoreBlockIndices()
    1702: @*/
    1703: PetscErrorCode  ISLocalToGlobalMappingGetBlockIndices(ISLocalToGlobalMapping ltog,const PetscInt **array)
    1704: {
    1707:   *array = ltog->indices;
    1708:   return 0;
    1709: }

    1711: /*@C
    1712:    ISLocalToGlobalMappingRestoreBlockIndices - Restore indices obtained with ISLocalToGlobalMappingGetBlockIndices()

    1714:    Not Collective

    1716:    Input Parameters:
    1717: + ltog - local to global mapping
    1718: - array - array of indices

    1720:    Level: advanced

    1722: .seealso: ISLocalToGlobalMappingCreate(), ISLocalToGlobalMappingApply(), ISLocalToGlobalMappingGetIndices()
    1723: @*/
    1724: PetscErrorCode  ISLocalToGlobalMappingRestoreBlockIndices(ISLocalToGlobalMapping ltog,const PetscInt **array)
    1725: {
    1729:   *array = NULL;
    1730:   return 0;
    1731: }

    1733: /*@C
    1734:    ISLocalToGlobalMappingConcatenate - Create a new mapping that concatenates a list of mappings

    1736:    Not Collective

    1738:    Input Parameters:
    1739: + comm - communicator for the new mapping, must contain the communicator of every mapping to concatenate
    1740: . n - number of mappings to concatenate
    1741: - ltogs - local to global mappings

    1743:    Output Parameter:
    1744: . ltogcat - new mapping

    1746:    Note: this currently always returns a mapping with block size of 1

    1748:    Developer Note: If all the input mapping have the same block size we could easily handle that as a special case

    1750:    Level: advanced

    1752: .seealso: ISLocalToGlobalMappingCreate()
    1753: @*/
    1754: PetscErrorCode ISLocalToGlobalMappingConcatenate(MPI_Comm comm,PetscInt n,const ISLocalToGlobalMapping ltogs[],ISLocalToGlobalMapping *ltogcat)
    1755: {
    1756:   PetscInt       i,cnt,m,*idx;

    1762:   for (cnt=0,i=0; i<n; i++) {
    1763:     ISLocalToGlobalMappingGetSize(ltogs[i],&m);
    1764:     cnt += m;
    1765:   }
    1766:   PetscMalloc1(cnt,&idx);
    1767:   for (cnt=0,i=0; i<n; i++) {
    1768:     const PetscInt *subidx;
    1769:     ISLocalToGlobalMappingGetSize(ltogs[i],&m);
    1770:     ISLocalToGlobalMappingGetIndices(ltogs[i],&subidx);
    1771:     PetscArraycpy(&idx[cnt],subidx,m);
    1772:     ISLocalToGlobalMappingRestoreIndices(ltogs[i],&subidx);
    1773:     cnt += m;
    1774:   }
    1775:   ISLocalToGlobalMappingCreate(comm,1,cnt,idx,PETSC_OWN_POINTER,ltogcat);
    1776:   return 0;
    1777: }

    1779: /*MC
    1780:       ISLOCALTOGLOBALMAPPINGBASIC - basic implementation of the ISLocalToGlobalMapping object. When ISGlobalToLocalMappingApply() is
    1781:                                     used this is good for only small and moderate size problems.

    1783:    Options Database Keys:
    1784: .   -islocaltoglobalmapping_type basic - select this method

    1786:    Level: beginner

    1788: .seealso:  ISLocalToGlobalMappingCreate(), ISLocalToGlobalMappingSetType(), ISLOCALTOGLOBALMAPPINGHASH
    1789: M*/
    1790: PETSC_EXTERN PetscErrorCode ISLocalToGlobalMappingCreate_Basic(ISLocalToGlobalMapping ltog)
    1791: {
    1792:   ltog->ops->globaltolocalmappingapply      = ISGlobalToLocalMappingApply_Basic;
    1793:   ltog->ops->globaltolocalmappingsetup      = ISGlobalToLocalMappingSetUp_Basic;
    1794:   ltog->ops->globaltolocalmappingapplyblock = ISGlobalToLocalMappingApplyBlock_Basic;
    1795:   ltog->ops->destroy                        = ISLocalToGlobalMappingDestroy_Basic;
    1796:   return 0;
    1797: }

    1799: /*MC
    1800:       ISLOCALTOGLOBALMAPPINGHASH - hash implementation of the ISLocalToGlobalMapping object. When ISGlobalToLocalMappingApply() is
    1801:                                     used this is good for large memory problems.

    1803:    Options Database Keys:
    1804: .   -islocaltoglobalmapping_type hash - select this method

    1806:    Notes:
    1807:     This is selected automatically for large problems if the user does not set the type.

    1809:    Level: beginner

    1811: .seealso:  ISLocalToGlobalMappingCreate(), ISLocalToGlobalMappingSetType(), ISLOCALTOGLOBALMAPPINGHASH
    1812: M*/
    1813: PETSC_EXTERN PetscErrorCode ISLocalToGlobalMappingCreate_Hash(ISLocalToGlobalMapping ltog)
    1814: {
    1815:   ltog->ops->globaltolocalmappingapply      = ISGlobalToLocalMappingApply_Hash;
    1816:   ltog->ops->globaltolocalmappingsetup      = ISGlobalToLocalMappingSetUp_Hash;
    1817:   ltog->ops->globaltolocalmappingapplyblock = ISGlobalToLocalMappingApplyBlock_Hash;
    1818:   ltog->ops->destroy                        = ISLocalToGlobalMappingDestroy_Hash;
    1819:   return 0;
    1820: }

    1822: /*@C
    1823:     ISLocalToGlobalMappingRegister -  Adds a method for applying a global to local mapping with an ISLocalToGlobalMapping

    1825:    Not Collective

    1827:    Input Parameters:
    1828: +  sname - name of a new method
    1829: -  routine_create - routine to create method context

    1831:    Notes:
    1832:    ISLocalToGlobalMappingRegister() may be called multiple times to add several user-defined mappings.

    1834:    Sample usage:
    1835: .vb
    1836:    ISLocalToGlobalMappingRegister("my_mapper",MyCreate);
    1837: .ve

    1839:    Then, your mapping can be chosen with the procedural interface via
    1840: $     ISLocalToGlobalMappingSetType(ltog,"my_mapper")
    1841:    or at runtime via the option
    1842: $     -islocaltoglobalmapping_type my_mapper

    1844:    Level: advanced

    1846: .seealso: ISLocalToGlobalMappingRegisterAll(), ISLocalToGlobalMappingRegisterDestroy(), ISLOCALTOGLOBALMAPPINGBASIC, ISLOCALTOGLOBALMAPPINGHASH

    1848: @*/
    1849: PetscErrorCode  ISLocalToGlobalMappingRegister(const char sname[],PetscErrorCode (*function)(ISLocalToGlobalMapping))
    1850: {
    1851:   ISInitializePackage();
    1852:   PetscFunctionListAdd(&ISLocalToGlobalMappingList,sname,function);
    1853:   return 0;
    1854: }

    1856: /*@C
    1857:    ISLocalToGlobalMappingSetType - Builds ISLocalToGlobalMapping for a particular global to local mapping approach.

    1859:    Logically Collective on ISLocalToGlobalMapping

    1861:    Input Parameters:
    1862: +  ltog - the ISLocalToGlobalMapping object
    1863: -  type - a known method

    1865:    Options Database Key:
    1866: .  -islocaltoglobalmapping_type  <method> - Sets the method; use -help for a list
    1867:     of available methods (for instance, basic or hash)

    1869:    Notes:
    1870:    See "petsc/include/petscis.h" for available methods

    1872:   Normally, it is best to use the ISLocalToGlobalMappingSetFromOptions() command and
    1873:   then set the ISLocalToGlobalMapping type from the options database rather than by using
    1874:   this routine.

    1876:   Level: intermediate

    1878:   Developer Note: ISLocalToGlobalMappingRegister() is used to add new types to ISLocalToGlobalMappingList from which they
    1879:   are accessed by ISLocalToGlobalMappingSetType().

    1881: .seealso: ISLocalToGlobalMappingType, ISLocalToGlobalMappingRegister(), ISLocalToGlobalMappingCreate(), ISLocalToGlobalMappingGetType()
    1882: @*/
    1883: PetscErrorCode  ISLocalToGlobalMappingSetType(ISLocalToGlobalMapping ltog, ISLocalToGlobalMappingType type)
    1884: {
    1885:   PetscBool      match;
    1886:   PetscErrorCode (*r)(ISLocalToGlobalMapping) = NULL;

    1891:   PetscObjectTypeCompare((PetscObject)ltog,type,&match);
    1892:   if (match) return 0;

    1894:   /* L2G maps defer type setup at globaltolocal calls, allow passing NULL here */
    1895:   if (type) {
    1896:     PetscFunctionListFind(ISLocalToGlobalMappingList,type,&r);
    1898:   }
    1899:   /* Destroy the previous private LTOG context */
    1900:   if (ltog->ops->destroy) {
    1901:     (*ltog->ops->destroy)(ltog);
    1902:     ltog->ops->destroy = NULL;
    1903:   }
    1904:   PetscObjectChangeTypeName((PetscObject)ltog,type);
    1905:   if (r) (*r)(ltog);
    1906:   return 0;
    1907: }

    1909: /*@C
    1910:    ISLocalToGlobalMappingGetType - Get the type of the l2g map

    1912:    Not Collective

    1914:    Input Parameter:
    1915: .  ltog - the ISLocalToGlobalMapping object

    1917:    Output Parameter:
    1918: .  type - the type

    1920:    Level: intermediate

    1922: .seealso: ISLocalToGlobalMappingType, ISLocalToGlobalMappingRegister(), ISLocalToGlobalMappingCreate(), ISLocalToGlobalMappingSetType()
    1923: @*/
    1924: PetscErrorCode  ISLocalToGlobalMappingGetType(ISLocalToGlobalMapping ltog, ISLocalToGlobalMappingType *type)
    1925: {
    1928:   *type = ((PetscObject)ltog)->type_name;
    1929:   return 0;
    1930: }

    1932: PetscBool ISLocalToGlobalMappingRegisterAllCalled = PETSC_FALSE;

    1934: /*@C
    1935:   ISLocalToGlobalMappingRegisterAll - Registers all of the local to global mapping components in the IS package.

    1937:   Not Collective

    1939:   Level: advanced

    1941: .seealso:  ISRegister(),  ISLocalToGlobalRegister()
    1942: @*/
    1943: PetscErrorCode  ISLocalToGlobalMappingRegisterAll(void)
    1944: {
    1945:   if (ISLocalToGlobalMappingRegisterAllCalled) return 0;
    1946:   ISLocalToGlobalMappingRegisterAllCalled = PETSC_TRUE;
    1947:   ISLocalToGlobalMappingRegister(ISLOCALTOGLOBALMAPPINGBASIC, ISLocalToGlobalMappingCreate_Basic);
    1948:   ISLocalToGlobalMappingRegister(ISLOCALTOGLOBALMAPPINGHASH, ISLocalToGlobalMappingCreate_Hash);
    1949:   return 0;
    1950: }