Date: 2022-03-31T04:47:39+00:00

Actual source code: data_ex.c

      1: /*
      2: Build a few basic tools to help with partitioned domains.

      4: 1)
      5: On each processor, have a DomainExchangerTopology.
      6: This is a doubly-connected edge list which enumerates the
      7: communication paths between connected processors. By numbering
      8: these paths we can always uniquely assign message identifers.

     10:         edge
     11:          10
     12: proc  --------->  proc
     13:  0    <--------    1
     14:          11
     15:         twin

     17: Eg: Proc 0 send to proc 1 with message id is 10. To receive the correct
     18: message, proc 1 looks for the edge connected to proc 0, and then the
     19: message id comes from the twin of that edge

     21: 2)
     22: A DomainExchangerArrayPacker.
     23: A little function which given a piece of data, will memcpy the data into
     24: an array (which will be sent to procs) into the correct place.

     26: On Proc 1 we sent data to procs 0,2,3. The data is on different lengths.
     27: All data gets jammed into single array. Need to "jam" data into correct locations
     28: The Packer knows how much is to going to each processor and keeps track of the inserts
     29: so as to avoid ever packing TOO much into one slot, and inevatbly corrupting some memory

     31: data to 0    data to 2       data to 3

     33: |--------|-----------------|--|

     35: User has to unpack message themselves. I can get you the pointer for each i
     36: entry, but you'll have to cast it to the appropriate data type.

     38: Phase A: Build topology

     40: Phase B: Define message lengths

     42: Phase C: Pack data

     44: Phase D: Send data

     46: + Constructor
     47: DMSwarmDataExCreate()
     48: + Phase A
     49: DMSwarmDataExTopologyInitialize()
     50: DMSwarmDataExTopologyAddNeighbour()
     51: DMSwarmDataExTopologyAddNeighbour()
     52: DMSwarmDataExTopologyFinalize()
     53: + Phase B
     54: DMSwarmDataExZeroAllSendCount()
     55: DMSwarmDataExAddToSendCount()
     56: DMSwarmDataExAddToSendCount()
     57: DMSwarmDataExAddToSendCount()
     58: + Phase C
     59: DMSwarmDataExPackInitialize()
     60: DMSwarmDataExPackData()
     61: DMSwarmDataExPackData()
     62: DMSwarmDataExPackFinalize()
     63: +Phase D
     64: DMSwarmDataExBegin()
     65:  ... perform any calculations ...
     66: DMSwarmDataExEnd()

     68: ... user calls any getters here ...

     70: */
     71: #include <petscvec.h>
     72: #include <petscmat.h>

     74: #include "../src/dm/impls/swarm/data_ex.h"

     76: const char *status_names[] = {"initialized", "finalized", "unknown"};

     78: PETSC_EXTERN PetscLogEvent DMSWARM_DataExchangerTopologySetup;
     79: PETSC_EXTERN PetscLogEvent DMSWARM_DataExchangerBegin;
     80: PETSC_EXTERN PetscLogEvent DMSWARM_DataExchangerEnd;
     81: PETSC_EXTERN PetscLogEvent DMSWARM_DataExchangerSendCount;
     82: PETSC_EXTERN PetscLogEvent DMSWARM_DataExchangerPack;

     84: PetscErrorCode DMSwarmDataExCreate(MPI_Comm comm,const PetscInt count, DMSwarmDataEx *ex)
     85: {
     86:   DMSwarmDataEx  d;

     88:   PetscNew(&d);
     89:   MPI_Comm_dup(comm,&d->comm);
     90:   MPI_Comm_rank(d->comm,&d->rank);

     92:   d->instance = count;

     94:   d->topology_status        = DEOBJECT_STATE_UNKNOWN;
     95:   d->message_lengths_status = DEOBJECT_STATE_UNKNOWN;
     96:   d->packer_status          = DEOBJECT_STATE_UNKNOWN;
     97:   d->communication_status   = DEOBJECT_STATE_UNKNOWN;

     99:   d->n_neighbour_procs = -1;
    100:   d->neighbour_procs   = NULL;

    102:   d->messages_to_be_sent      = NULL;
    103:   d->message_offsets          = NULL;
    104:   d->messages_to_be_recvieved = NULL;

    106:   d->unit_message_size   = (size_t)-1;
    107:   d->send_message        = NULL;
    108:   d->send_message_length = -1;
    109:   d->recv_message        = NULL;
    110:   d->recv_message_length = -1;
    111:   d->total_pack_cnt      = -1;
    112:   d->pack_cnt            = NULL;

    114:   d->send_tags = NULL;
    115:   d->recv_tags = NULL;

    117:   d->_stats    = NULL;
    118:   d->_requests = NULL;
    119:   *ex = d;
    120:   return 0;
    121: }

    123: /*
    124:     This code is horrible, who let it get into main.

    126:     Should be printing to a viewer, should not be using PETSC_COMM_WORLD

    128: */
    129: PetscErrorCode DMSwarmDataExView(DMSwarmDataEx d)
    130: {
    131:   PetscMPIInt    p;

    133:   PetscPrintf( PETSC_COMM_WORLD, "DMSwarmDataEx: instance=%D\n",d->instance);
    134:   PetscPrintf( PETSC_COMM_WORLD, "  topology status:        %s \n", status_names[d->topology_status]);
    135:   PetscPrintf( PETSC_COMM_WORLD, "  message lengths status: %s \n", status_names[d->message_lengths_status]);
    136:   PetscPrintf( PETSC_COMM_WORLD, "  packer status status:   %s \n", status_names[d->packer_status]);
    137:   PetscPrintf( PETSC_COMM_WORLD, "  communication status:   %s \n", status_names[d->communication_status]);

    139:   if (d->topology_status == DEOBJECT_FINALIZED) {
    140:     PetscPrintf( PETSC_COMM_WORLD, "  Topology:\n");
    141:     PetscSynchronizedPrintf( PETSC_COMM_WORLD, "    [%d] neighbours: %d \n", d->rank, d->n_neighbour_procs);
    142:     for (p=0; p<d->n_neighbour_procs; p++) {
    143:       PetscSynchronizedPrintf( PETSC_COMM_WORLD, "    [%d]   neighbour[%d] = %d \n", d->rank, p, d->neighbour_procs[p]);
    144:     }
    145:     PetscSynchronizedFlush(PETSC_COMM_WORLD,stdout);
    146:   }

    148:   if (d->message_lengths_status == DEOBJECT_FINALIZED) {
    149:     PetscPrintf( PETSC_COMM_WORLD, "  Message lengths:\n");
    150:     PetscSynchronizedPrintf( PETSC_COMM_WORLD, "    [%d] atomic size: %ld \n", d->rank, (long int)d->unit_message_size);
    151:     for (p=0; p<d->n_neighbour_procs; p++) {
    152:       PetscSynchronizedPrintf( PETSC_COMM_WORLD, "    [%d] >>>>> ( %D units :: tag = %d) >>>>> [%d] \n", d->rank, d->messages_to_be_sent[p], d->send_tags[p], d->neighbour_procs[p]);
    153:     }
    154:     for (p=0; p<d->n_neighbour_procs; p++) {
    155:       PetscSynchronizedPrintf( PETSC_COMM_WORLD, "    [%d] <<<<< ( %D units :: tag = %d) <<<<< [%d] \n", d->rank, d->messages_to_be_recvieved[p], d->recv_tags[p], d->neighbour_procs[p]);
    156:     }
    157:     PetscSynchronizedFlush(PETSC_COMM_WORLD,stdout);
    158:   }
    159:   if (d->packer_status == DEOBJECT_FINALIZED) {}
    160:   if (d->communication_status == DEOBJECT_FINALIZED) {}
    161:   return 0;
    162: }

    164: PetscErrorCode DMSwarmDataExDestroy(DMSwarmDataEx d)
    165: {
    166:   MPI_Comm_free(&d->comm);
    167:   if (d->neighbour_procs) PetscFree(d->neighbour_procs);
    168:   if (d->messages_to_be_sent) PetscFree(d->messages_to_be_sent);
    169:   if (d->message_offsets) PetscFree(d->message_offsets);
    170:   if (d->messages_to_be_recvieved) PetscFree(d->messages_to_be_recvieved);
    171:   if (d->send_message) PetscFree(d->send_message);
    172:   if (d->recv_message) PetscFree(d->recv_message);
    173:   if (d->pack_cnt) PetscFree(d->pack_cnt);
    174:   if (d->send_tags) PetscFree(d->send_tags);
    175:   if (d->recv_tags) PetscFree(d->recv_tags);
    176:   if (d->_stats) PetscFree(d->_stats);
    177:   if (d->_requests) PetscFree(d->_requests);
    178:   PetscFree(d);
    179:   return 0;
    180: }

    182: /* === Phase A === */

    184: PetscErrorCode DMSwarmDataExTopologyInitialize(DMSwarmDataEx d)
    185: {
    186:   d->topology_status = DEOBJECT_INITIALIZED;
    187:   d->n_neighbour_procs = 0;
    188:   PetscFree(d->neighbour_procs);
    189:   PetscFree(d->messages_to_be_sent);
    190:   PetscFree(d->message_offsets);
    191:   PetscFree(d->messages_to_be_recvieved);
    192:   PetscFree(d->pack_cnt);
    193:   PetscFree(d->send_tags);
    194:   PetscFree(d->recv_tags);
    195:   return 0;
    196: }

    198: PetscErrorCode DMSwarmDataExTopologyAddNeighbour(DMSwarmDataEx d,const PetscMPIInt proc_id)
    199: {
    200:   PetscMPIInt    n,found;
    201:   PetscMPIInt    size;

    206:   /* error on negative entries */
    208:   /* error on ranks larger than number of procs in communicator */
    209:   MPI_Comm_size(d->comm,&size);
    211:   if (d->n_neighbour_procs == 0) PetscMalloc1(1, &d->neighbour_procs);
    212:   /* check for proc_id */
    213:   found = 0;
    214:   for (n = 0; n < d->n_neighbour_procs; n++) {
    215:     if (d->neighbour_procs[n] == proc_id) {
    216:       found  = 1;
    217:     }
    218:   }
    219:   if (found == 0) { /* add it to list */
    220:     PetscRealloc(sizeof(PetscMPIInt)*(d->n_neighbour_procs+1), &d->neighbour_procs);
    221:     d->neighbour_procs[ d->n_neighbour_procs ] = proc_id;
    222:     d->n_neighbour_procs++;
    223:   }
    224:   return 0;
    225: }

    227: /*
    228: counter: the index of the communication object
    229: N: the number of processors
    230: r0: rank of sender
    231: r1: rank of receiver

    233: procs = { 0, 1, 2, 3 }

    235: 0 ==> 0         e=0
    236: 0 ==> 1         e=1
    237: 0 ==> 2         e=2
    238: 0 ==> 3         e=3

    240: 1 ==> 0         e=4
    241: 1 ==> 1         e=5
    242: 1 ==> 2         e=6
    243: 1 ==> 3         e=7

    245: 2 ==> 0         e=8
    246: 2 ==> 1         e=9
    247: 2 ==> 2         e=10
    248: 2 ==> 3         e=11

    250: 3 ==> 0         e=12
    251: 3 ==> 1         e=13
    252: 3 ==> 2         e=14
    253: 3 ==> 3         e=15

    255: If we require that proc A sends to proc B, then the SEND tag index will be given by
    256:   N * rank(A) + rank(B) + offset
    257: If we require that proc A will receive from proc B, then the RECV tag index will be given by
    258:   N * rank(B) + rank(A) + offset

    260: */
    261: static void _get_tags(PetscInt counter, PetscMPIInt N, PetscMPIInt r0,PetscMPIInt r1, PetscMPIInt *_st, PetscMPIInt *_rt)
    262: {
    263:   PetscMPIInt st,rt;

    265:   st = N*r0 + r1   +   N*N*counter;
    266:   rt = N*r1 + r0   +   N*N*counter;
    267:   *_st = st;
    268:   *_rt = rt;
    269: }

    271: /*
    272: Makes the communication map symmetric
    273: */
    274: PetscErrorCode _DMSwarmDataExCompleteCommunicationMap(MPI_Comm comm,PetscMPIInt n,PetscMPIInt proc_neighbours[],PetscMPIInt *n_new,PetscMPIInt **proc_neighbours_new)
    275: {
    276:   Mat               A;
    277:   PetscInt          i,j,nc;
    278:   PetscInt          n_, *proc_neighbours_;
    279:   PetscInt          rank_;
    280:   PetscMPIInt       size,  rank;
    281:   PetscScalar       *vals;
    282:   const PetscInt    *cols;
    283:   const PetscScalar *red_vals;
    284:   PetscMPIInt       _n_new, *_proc_neighbours_new;

    286:   n_ = n;
    287:   PetscMalloc(sizeof(PetscInt) * n_, &proc_neighbours_);
    288:   for (i = 0; i < n_; ++i) {
    289:     proc_neighbours_[i] = proc_neighbours[i];
    290:   }
    291:   MPI_Comm_size(comm,&size);
    292:   MPI_Comm_rank(comm,&rank);
    293:   rank_ = rank;

    295:   MatCreate(comm,&A);
    296:   MatSetSizes(A,PETSC_DECIDE,PETSC_DECIDE,size,size);
    297:   MatSetType(A,MATAIJ);
    298:   MatSeqAIJSetPreallocation(A,1,NULL);
    299:   MatMPIAIJSetPreallocation(A,n_,NULL,n_,NULL);
    300:   MatSetOption(A, MAT_NEW_NONZERO_ALLOCATION_ERR, PETSC_FALSE);
    301:   /* Build original map */
    302:   PetscMalloc1(n_, &vals);
    303:   for (i = 0; i < n_; ++i) {
    304:     vals[i] = 1.0;
    305:   }
    306:   MatSetValues( A, 1,&rank_, n_,proc_neighbours_, vals, INSERT_VALUES);
    307:   MatAssemblyBegin(A,MAT_FLUSH_ASSEMBLY);
    308:   MatAssemblyEnd(A,MAT_FLUSH_ASSEMBLY);
    309:   /* Now force all other connections if they are not already there */
    310:   /* It's more efficient to do them all at once */
    311:   for (i = 0; i < n_; ++i) {
    312:     vals[i] = 2.0;
    313:   }
    314:   MatSetValues( A, n_,proc_neighbours_, 1,&rank_, vals, INSERT_VALUES);
    315:   MatAssemblyBegin(A,MAT_FINAL_ASSEMBLY);
    316:   MatAssemblyEnd(A,MAT_FINAL_ASSEMBLY);
    317: /*
    318:   PetscViewerPushFormat(PETSC_VIEWER_STDOUT_WORLD,PETSC_VIEWER_ASCII_INFO);
    319:   MatView(A,PETSC_VIEWER_STDOUT_WORLD);
    320:   PetscViewerPopFormat(PETSC_VIEWER_STDOUT_WORLD);
    321: */
    322:   if ((n_new != NULL) && (proc_neighbours_new != NULL)) {
    323:     MatGetRow(A, rank_, &nc, &cols, &red_vals);
    324:     _n_new = (PetscMPIInt) nc;
    325:     PetscMalloc1(_n_new, &_proc_neighbours_new);
    326:     for (j = 0; j < nc; ++j) {
    327:       _proc_neighbours_new[j] = (PetscMPIInt)cols[j];
    328:     }
    329:     MatRestoreRow( A, rank_, &nc, &cols, &red_vals);
    330:     *n_new               = (PetscMPIInt)_n_new;
    331:     *proc_neighbours_new = (PetscMPIInt*)_proc_neighbours_new;
    332:   }
    333:   MatDestroy(&A);
    334:   PetscFree(vals);
    335:   PetscFree(proc_neighbours_);
    336:   MPI_Barrier(comm);
    337:   return 0;
    338: }

    340: PetscErrorCode DMSwarmDataExTopologyFinalize(DMSwarmDataEx d)
    341: {
    342:   PetscMPIInt    symm_nn, *symm_procs, r0,n,st,rt, size;

    346:   PetscLogEventBegin(DMSWARM_DataExchangerTopologySetup,0,0,0,0);
    347:   /* given information about all my neighbours, make map symmetric */
    348:   _DMSwarmDataExCompleteCommunicationMap( d->comm,d->n_neighbour_procs,d->neighbour_procs, &symm_nn, &symm_procs);
    349:   /* update my arrays */
    350:   PetscFree(d->neighbour_procs);
    351:   d->n_neighbour_procs = symm_nn;
    352:   d->neighbour_procs   = symm_procs;
    353:   /* allocates memory */
    354:   if (!d->messages_to_be_sent) PetscMalloc1(d->n_neighbour_procs+1, &d->messages_to_be_sent);
    355:   if (!d->message_offsets) PetscMalloc1(d->n_neighbour_procs+1, &d->message_offsets);
    356:   if (!d->messages_to_be_recvieved) PetscMalloc1(d->n_neighbour_procs+1, &d->messages_to_be_recvieved);
    357:   if (!d->pack_cnt) PetscMalloc(sizeof(PetscInt) * d->n_neighbour_procs, &d->pack_cnt);
    358:   if (!d->_stats) PetscMalloc(sizeof(MPI_Status) * 2*d->n_neighbour_procs, &d->_stats);
    359:   if (!d->_requests) PetscMalloc(sizeof(MPI_Request) * 2*d->n_neighbour_procs, &d->_requests);
    360:   if (!d->send_tags) PetscMalloc(sizeof(int) * d->n_neighbour_procs, &d->send_tags);
    361:   if (!d->recv_tags) PetscMalloc(sizeof(int) * d->n_neighbour_procs, &d->recv_tags);
    362:   /* compute message tags */
    363:   MPI_Comm_size(d->comm,&size);
    364:   r0 = d->rank;
    365:   for (n = 0; n < d->n_neighbour_procs; ++n) {
    366:     PetscMPIInt r1 = d->neighbour_procs[n];

    368:     _get_tags( d->instance, size, r0,r1, &st, &rt);
    369:     d->send_tags[n] = (int)st;
    370:     d->recv_tags[n] = (int)rt;
    371:   }
    372:   d->topology_status = DEOBJECT_FINALIZED;
    373:   PetscLogEventEnd(DMSWARM_DataExchangerTopologySetup,0,0,0,0);
    374:   return 0;
    375: }

    377: /* === Phase B === */
    378: PetscErrorCode _DMSwarmDataExConvertProcIdToLocalIndex(DMSwarmDataEx de,PetscMPIInt proc_id,PetscMPIInt *local)
    379: {
    380:   PetscMPIInt i,np;

    382:   np = de->n_neighbour_procs;
    383:   *local = -1;
    384:   for (i = 0; i < np; ++i) {
    385:     if (proc_id == de->neighbour_procs[i]) {
    386:       *local = i;
    387:       break;
    388:     }
    389:   }
    390:   return 0;
    391: }

    393: PetscErrorCode DMSwarmDataExInitializeSendCount(DMSwarmDataEx de)
    394: {
    395:   PetscMPIInt    i;

    398:   PetscLogEventBegin(DMSWARM_DataExchangerSendCount,0,0,0,0);
    399:   de->message_lengths_status = DEOBJECT_INITIALIZED;
    400:   for (i = 0; i < de->n_neighbour_procs; ++i) {
    401:     de->messages_to_be_sent[i] = 0;
    402:   }
    403:   return 0;
    404: }

    406: /*
    407: 1) only allows counters to be set on neighbouring cpus
    408: */
    409: PetscErrorCode DMSwarmDataExAddToSendCount(DMSwarmDataEx de,const PetscMPIInt proc_id,const PetscInt count)
    410: {
    411:   PetscMPIInt    local_val;

    416:   _DMSwarmDataExConvertProcIdToLocalIndex( de, proc_id, &local_val);

    419:   de->messages_to_be_sent[local_val] = de->messages_to_be_sent[local_val] + count;
    420:   return 0;
    421: }

    423: PetscErrorCode DMSwarmDataExFinalizeSendCount(DMSwarmDataEx de)
    424: {

    427:   de->message_lengths_status = DEOBJECT_FINALIZED;
    428:   PetscLogEventEnd(DMSWARM_DataExchangerSendCount,0,0,0,0);
    429:   return 0;
    430: }

    432: /* === Phase C === */
    433: /*
    434:   zero out all send counts
    435:   free send and recv buffers
    436:   zeros out message length
    437:   zeros out all counters
    438:   zero out packed data counters
    439: */
    440: PetscErrorCode _DMSwarmDataExInitializeTmpStorage(DMSwarmDataEx de)
    441: {
    442:   PetscMPIInt    i, np;

    444:   np = de->n_neighbour_procs;
    445:   for (i = 0; i < np; ++i) {
    446:     /*  de->messages_to_be_sent[i] = -1; */
    447:     de->messages_to_be_recvieved[i] = -1;
    448:   }
    449:   PetscFree(de->send_message);
    450:   PetscFree(de->recv_message);
    451:   return 0;
    452: }

    454: /*
    455:    Zeros out pack data counters
    456:    Ensures mesaage length is set
    457:    Checks send counts properly initialized
    458:    allocates space for pack data
    459: */
    460: PetscErrorCode DMSwarmDataExPackInitialize(DMSwarmDataEx de,size_t unit_message_size)
    461: {
    462:   PetscMPIInt    i,np;
    463:   PetscInt       total;

    467:   PetscLogEventBegin(DMSWARM_DataExchangerPack,0,0,0,0);
    468:   de->packer_status = DEOBJECT_INITIALIZED;
    469:   _DMSwarmDataExInitializeTmpStorage(de);
    470:   np = de->n_neighbour_procs;
    471:   de->unit_message_size = unit_message_size;
    472:   total = 0;
    473:   for (i = 0; i < np; ++i) {
    474:     if (de->messages_to_be_sent[i] == -1) {
    475:       PetscMPIInt proc_neighour = de->neighbour_procs[i];
    476:       SETERRQ( PETSC_COMM_SELF, PETSC_ERR_ORDER, "Messages_to_be_sent[neighbour_proc=%d] is un-initialised. Call DMSwarmDataExSetSendCount() first", (int)proc_neighour);
    477:     }
    478:     total = total + de->messages_to_be_sent[i];
    479:   }
    480:   /* create space for the data to be sent */
    481:   PetscMalloc(unit_message_size * (total + 1), &de->send_message);
    482:   /* initialize memory */
    483:   PetscMemzero(de->send_message, unit_message_size * (total + 1));
    484:   /* set total items to send */
    485:   de->send_message_length = total;
    486:   de->message_offsets[0] = 0;
    487:   total = de->messages_to_be_sent[0];
    488:   for (i = 1; i < np; ++i) {
    489:     de->message_offsets[i] = total;
    490:     total = total + de->messages_to_be_sent[i];
    491:   }
    492:   /* init the packer counters */
    493:   de->total_pack_cnt = 0;
    494:   for (i = 0; i < np; ++i) {
    495:     de->pack_cnt[i] = 0;
    496:   }
    497:   return 0;
    498: }

    500: /*
    501:     Ensures data gets been packed appropriately and no overlaps occur
    502: */
    503: PetscErrorCode DMSwarmDataExPackData(DMSwarmDataEx de,PetscMPIInt proc_id,PetscInt n,void *data)
    504: {
    505:   PetscMPIInt    local;
    506:   PetscInt       insert_location;
    507:   void           *dest;

    513:   _DMSwarmDataExConvertProcIdToLocalIndex( de, proc_id, &local);
    516:               (int)proc_id, de->messages_to_be_sent[local], n+de->pack_cnt[local]);

    518:   /* copy memory */
    519:   insert_location = de->message_offsets[local] + de->pack_cnt[local];
    520:   dest = ((char*)de->send_message) + de->unit_message_size*insert_location;
    521:   PetscMemcpy(dest, data, de->unit_message_size * n);
    522:   /* increment counter */
    523:   de->pack_cnt[local] = de->pack_cnt[local] + n;
    524:   return 0;
    525: }

    527: /*
    528: *) Ensures all data has been packed
    529: */
    530: PetscErrorCode DMSwarmDataExPackFinalize(DMSwarmDataEx de)
    531: {
    532:   PetscMPIInt    i,np;
    533:   PetscInt       total;

    536:   np = de->n_neighbour_procs;
    537:   for (i = 0; i < np; ++i) {
    539:                 (int)de->neighbour_procs[i], de->messages_to_be_sent[i], de->pack_cnt[i]);
    540:   }
    541:   /* init */
    542:   for (i = 0; i < np; ++i) {
    543:     de->messages_to_be_recvieved[i] = -1;
    544:   }
    545:   /* figure out the recv counts here */
    546:   for (i = 0; i < np; ++i) {
    547:     MPI_Isend(&de->messages_to_be_sent[i], 1, MPIU_INT, de->neighbour_procs[i], de->send_tags[i], de->comm, &de->_requests[i]);
    548:   }
    549:   for (i = 0; i < np; ++i) {
    550:     MPI_Irecv(&de->messages_to_be_recvieved[i], 1, MPIU_INT, de->neighbour_procs[i], de->recv_tags[i], de->comm, &de->_requests[np+i]);
    551:   }
    552:   MPI_Waitall(2*np, de->_requests, de->_stats);
    553:   /* create space for the data to be recvieved */
    554:   total = 0;
    555:   for (i = 0; i < np; ++i) {
    556:     total = total + de->messages_to_be_recvieved[i];
    557:   }
    558:   PetscMalloc(de->unit_message_size * (total + 1), &de->recv_message);
    559:   /* initialize memory */
    560:   PetscMemzero(de->recv_message, de->unit_message_size * (total + 1));
    561:   /* set total items to receive */
    562:   de->recv_message_length = total;
    563:   de->packer_status = DEOBJECT_FINALIZED;
    564:   de->communication_status = DEOBJECT_INITIALIZED;
    565:   PetscLogEventEnd(DMSWARM_DataExchangerPack,0,0,0,0);
    566:   return 0;
    567: }

    569: /* do the actual message passing */
    570: PetscErrorCode DMSwarmDataExBegin(DMSwarmDataEx de)
    571: {
    572:   PetscMPIInt    i,np;
    573:   void           *dest;
    574:   PetscInt       length;

    581:   PetscLogEventBegin(DMSWARM_DataExchangerBegin,0,0,0,0);
    582:   np = de->n_neighbour_procs;
    583:   /* == NON BLOCKING == */
    584:   for (i = 0; i < np; ++i) {
    585:     length = de->messages_to_be_sent[i] * de->unit_message_size;
    586:     dest = ((char*)de->send_message) + de->unit_message_size * de->message_offsets[i];
    587:     MPI_Isend( dest, length, MPI_CHAR, de->neighbour_procs[i], de->send_tags[i], de->comm, &de->_requests[i]);
    588:   }
    589:   PetscLogEventEnd(DMSWARM_DataExchangerBegin,0,0,0,0);
    590:   return 0;
    591: }

    593: /* do the actual message passing now */
    594: PetscErrorCode DMSwarmDataExEnd(DMSwarmDataEx de)
    595: {
    596:   PetscMPIInt    i,np;
    597:   PetscInt       total;
    598:   PetscInt       *message_recv_offsets;
    599:   void           *dest;
    600:   PetscInt       length;

    604:   PetscLogEventBegin(DMSWARM_DataExchangerEnd,0,0,0,0);
    605:   np = de->n_neighbour_procs;
    606:   PetscMalloc1(np+1, &message_recv_offsets);
    607:   message_recv_offsets[0] = 0;
    608:   total = de->messages_to_be_recvieved[0];
    609:   for (i = 1; i < np; ++i) {
    610:     message_recv_offsets[i] = total;
    611:     total = total + de->messages_to_be_recvieved[i];
    612:   }
    613:   /* == NON BLOCKING == */
    614:   for (i = 0; i < np; ++i) {
    615:     length = de->messages_to_be_recvieved[i] * de->unit_message_size;
    616:     dest = ((char*)de->recv_message) + de->unit_message_size * message_recv_offsets[i];
    617:     MPI_Irecv( dest, length, MPI_CHAR, de->neighbour_procs[i], de->recv_tags[i], de->comm, &de->_requests[np+i]);
    618:   }
    619:   MPI_Waitall( 2*np, de->_requests, de->_stats);
    620:   PetscFree(message_recv_offsets);
    621:   de->communication_status = DEOBJECT_FINALIZED;
    622:   PetscLogEventEnd(DMSWARM_DataExchangerEnd,0,0,0,0);
    623:   return 0;
    624: }

    626: PetscErrorCode DMSwarmDataExGetSendData(DMSwarmDataEx de,PetscInt *length,void **send)
    627: {
    629:   *length = de->send_message_length;
    630:   *send   = de->send_message;
    631:   return 0;
    632: }

    634: PetscErrorCode DMSwarmDataExGetRecvData(DMSwarmDataEx de,PetscInt *length,void **recv)
    635: {
    637:   *length = de->recv_message_length;
    638:   *recv   = de->recv_message;
    639:   return 0;
    640: }

    642: PetscErrorCode DMSwarmDataExTopologyGetNeighbours(DMSwarmDataEx de,PetscMPIInt *n,PetscMPIInt *neigh[])
    643: {
    644:   if (n)     {*n     = de->n_neighbour_procs;}
    645:   if (neigh) {*neigh = de->neighbour_procs;}
    646:   return 0;
    647: }