Date: 2022-03-31T04:47:38+00:00

Actual source code: network.c

      1: #include <petsc/private/dmnetworkimpl.h>

      3: /*
      4:   Creates the component header and value objects for a network point
      5: */
      6: static PetscErrorCode SetUpNetworkHeaderComponentValue(DM dm,DMNetworkComponentHeader header,DMNetworkComponentValue cvalue)
      7: {
      8:   /* Allocate arrays for component information */
      9:   PetscCalloc5(header->maxcomps,&header->size,header->maxcomps,&header->key,header->maxcomps,&header->offset,header->maxcomps,&header->nvar,header->maxcomps,&header->offsetvarrel);
     10:   PetscCalloc1(header->maxcomps,&cvalue->data);

     12:   /* The size of the header is the size of struct _p_DMNetworkComponentHeader. Since the struct contains PetscInt pointers we cannot use sizeof(struct). So, we need to explicitly calculate the size.
     13:    If the data header struct changes then this header size calculation needs to be updated. */
     14:   header->hsize = sizeof(struct _p_DMNetworkComponentHeader) + 5*header->maxcomps*sizeof(PetscInt);
     15:   header->hsize /= sizeof(DMNetworkComponentGenericDataType);
     16:   return 0;
     17: }

     19: /*@
     20:   DMNetworkGetPlex - Gets the Plex DM associated with this network DM

     22:   Not collective

     24:   Input Parameters:
     25: . dm - the dm object

     27:   Output Parameters:
     28: . plexdm - the plex dm object

     30:   Level: Advanced

     32: .seealso: DMNetworkCreate()
     33: @*/
     34: PetscErrorCode DMNetworkGetPlex(DM dm,DM *plexdm)
     35: {
     36:   DM_Network *network = (DM_Network*)dm->data;

     38:   *plexdm = network->plex;
     39:   return 0;
     40: }

     42: /*@
     43:   DMNetworkGetNumSubNetworks - Gets the the number of subnetworks

     45:   Not collective

     47:   Input Parameter:
     48: . dm - the dm object

     50:   Output Parameters:
     51: + nsubnet - local number of subnetworks
     52: - Nsubnet - global number of subnetworks

     54:   Level: beginner

     56: .seealso: DMNetworkCreate(), DMNetworkSetNumSubNetworks()
     57: @*/
     58: PetscErrorCode DMNetworkGetNumSubNetworks(DM dm,PetscInt *nsubnet,PetscInt *Nsubnet)
     59: {
     60:   DM_Network *network = (DM_Network*)dm->data;

     62:   if (nsubnet) *nsubnet = network->nsubnet;
     63:   if (Nsubnet) *Nsubnet = network->Nsubnet;
     64:   return 0;
     65: }

     67: /*@
     68:   DMNetworkSetNumSubNetworks - Sets the number of subnetworks

     70:   Collective on dm

     72:   Input Parameters:
     73: + dm - the dm object
     74: . nsubnet - local number of subnetworks
     75: - Nsubnet - global number of subnetworks

     77:    Level: beginner

     79: .seealso: DMNetworkCreate(), DMNetworkGetNumSubNetworks()
     80: @*/
     81: PetscErrorCode DMNetworkSetNumSubNetworks(DM dm,PetscInt nsubnet,PetscInt Nsubnet)
     82: {
     83:   DM_Network     *network = (DM_Network*)dm->data;

     91:   if (Nsubnet == PETSC_DECIDE) {
     93:     MPIU_Allreduce(&nsubnet,&Nsubnet,1,MPIU_INT,MPI_SUM,PetscObjectComm((PetscObject)dm));
     94:   }

     97:   network->Nsubnet  = Nsubnet;
     98:   network->nsubnet  = 0;       /* initia value; will be determind by DMNetworkAddSubnetwork() */
     99:   PetscCalloc1(Nsubnet,&network->subnet);

    101:   /* num of shared vertices */
    102:   network->nsvtx = 0;
    103:   network->Nsvtx = 0;
    104:   return 0;
    105: }

    107: /*@
    108:   DMNetworkAddSubnetwork - Add a subnetwork

    110:   Collective on dm

    112:   Input Parameters:
    113: + dm - the dm object
    114: . name - name of the subnetwork
    115: . ne - number of local edges of this subnetwork
    116: - edgelist - list of edges for this subnetwork, this is a one dimensional array with pairs of entries being the two vertices (in global numbering of the vertices) of each edge,
    117: $            [first vertex of first edge, second vertex of first edge, first vertex of second edge, second vertex of second edge, etc]

    119:   Output Parameters:
    120: . netnum - global index of the subnetwork

    122:   Notes:
    123:   There is no copy involved in this operation, only the pointer is referenced. The edgelist should
    124:   not be destroyed before the call to DMNetworkLayoutSetUp()

    126:   A network can comprise of a single subnetwork OR multiple subnetworks. For a single subnetwork, the subnetwork can be read either in serial or parallel. For a multiple subnetworks,
    127:   each subnetwork topology needs to be set on a unique rank and the communicator size needs to be at least equal to the number of subnetworks.

    129:   Level: beginner

    131:   Example usage:
    132:   Consider the following networks:
    133:   1) A sigle subnetwork:
    134: .vb
    135:  network 0:
    136:  rank[0]:
    137:    v0 --> v2; v1 --> v2
    138:  rank[1]:
    139:    v3 --> v5; v4 --> v5
    140: .ve

    142:  The resulting input
    143:  network 0:
    144:  rank[0]:
    145:    ne = 2
    146:    edgelist = [0 2 | 1 2]
    147:  rank[1]:
    148:    ne = 2
    149:    edgelist = [3 5 | 4 5]

    151:   2) Two subnetworks:
    152: .vb
    153:  subnetwork 0:
    154:  rank[0]:
    155:    v0 --> v2; v2 --> v1; v1 --> v3;
    156:  subnetwork 1:
    157:  rank[1]:
    158:    v0 --> v3; v3 --> v2; v2 --> v1;
    159: .ve

    161:  The resulting input
    162:  subnetwork 0:
    163:  rank[0]:
    164:    ne = 3
    165:    edgelist = [0 2 | 2 1 | 1 3]
    166:  rank[1]:
    167:    ne = 0
    168:    edgelist = NULL

    170:  subnetwork 1:
    171:  rank[0]:
    172:    ne = 0
    173:    edgelist = NULL
    174:  rank[1]:
    175:    edgelist = [0 3 | 3 2 | 2 1]

    177: .seealso: DMNetworkCreate(), DMNetworkSetNumSubnetworks()
    178: @*/
    179: PetscErrorCode DMNetworkAddSubnetwork(DM dm,const char* name,PetscInt ne,PetscInt edgelist[],PetscInt *netnum)
    180: {
    181:   DM_Network     *network = (DM_Network*)dm->data;
    182:   PetscInt       i,Nedge,j,Nvtx,nvtx,nvtx_min=-1,nvtx_max=0;
    183:   PetscBT        table;

    185:   for (i=0; i<ne; i++) {
    187:   }

    189:   i = 0;
    190:   if (ne) nvtx_min = nvtx_max = edgelist[0];
    191:   for (j=0; j<ne; j++) {
    192:     nvtx_min = PetscMin(nvtx_min, edgelist[i]);
    193:     nvtx_max = PetscMax(nvtx_max, edgelist[i]);
    194:     i++;
    195:     nvtx_min = PetscMin(nvtx_min, edgelist[i]);
    196:     nvtx_max = PetscMax(nvtx_max, edgelist[i]);
    197:     i++;
    198:   }
    199:   Nvtx = nvtx_max - nvtx_min + 1; /* approximated total local nvtx for this subnet */

    201:   /* Get exact local nvtx for this subnet: counting local values between nvtx_min and nvtx_max */
    202:   PetscBTCreate(Nvtx,&table);
    203:   PetscBTMemzero(Nvtx,table);
    204:   i = 0;
    205:   for (j=0; j<ne; j++) {
    206:     PetscBTSet(table,edgelist[i++]-nvtx_min);
    207:     PetscBTSet(table,edgelist[i++]-nvtx_min);
    208:   }
    209:   nvtx = 0;
    210:   for (j=0; j<Nvtx; j++) {
    211:     if (PetscBTLookup(table,j)) nvtx++;
    212:   }

    214:   /* Get global total Nvtx = max(edgelist[])+1 for this subnet */
    215:   MPIU_Allreduce(&nvtx_max,&Nvtx,1,MPIU_INT,MPI_MAX,PetscObjectComm((PetscObject)dm));
    216:   Nvtx++;
    217:   PetscBTDestroy(&table);

    219:   /* Get global total Nedge for this subnet */
    220:   MPIU_Allreduce(&ne,&Nedge,1,MPIU_INT,MPI_SUM,PetscObjectComm((PetscObject)dm));

    222:   i = network->nsubnet;
    223:   if (name) {
    224:     PetscStrcpy(network->subnet[i].name,name);
    225:   }
    226:   network->subnet[i].nvtx     = nvtx; /* include ghost vertices */
    227:   network->subnet[i].nedge    = ne;
    228:   network->subnet[i].edgelist = edgelist;
    229:   network->subnet[i].Nvtx     = Nvtx;
    230:   network->subnet[i].Nedge    = Nedge;

    232:   /* ----------------------------------------------------------
    233:    p=v or e;
    234:    subnet[0].pStart   = 0
    235:    subnet[i+1].pStart = subnet[i].pEnd = subnet[i].pStart + (nE[i] or NV[i])
    236:    ----------------------------------------------------------------------- */
    237:   /* GLOBAL subnet[].vStart and vEnd, used by DMNetworkLayoutSetUp() */
    238:   network->subnet[i].vStart = network->NVertices;
    239:   network->subnet[i].vEnd   = network->subnet[i].vStart + network->subnet[i].Nvtx; /* global vEnd of subnet[i] */

    241:   network->nVertices += nvtx; /* include ghost vertices */
    242:   network->NVertices += network->subnet[i].Nvtx;

    244:   /* LOCAL subnet[].eStart and eEnd, used by DMNetworkLayoutSetUp() */
    245:   network->subnet[i].eStart = network->nEdges;
    246:   network->subnet[i].eEnd   = network->subnet[i].eStart + ne;
    247:   network->nEdges += ne;
    248:   network->NEdges += network->subnet[i].Nedge;

    250:   PetscStrcpy(network->subnet[i].name,name);
    251:   if (netnum) *netnum = network->nsubnet;
    252:   network->nsubnet++;
    253:   return 0;
    254: }

    256: /*@C
    257:   DMNetworkSharedVertexGetInfo - Get info of a shared vertex struct, see petsc/private/dmnetworkimpl.h

    259:   Not collective

    261:   Input Parameters:
    262: + dm - the DM object
    263: - v - vertex point

    265:   Output Parameters:
    266: + gidx - global number of this shared vertex in the internal dmplex
    267: . n - number of subnetworks that share this vertex
    268: - sv - array of size n: sv[2*i,2*i+1]=(net[i], idx[i]), i=0,...,n-1

    270:   Level: intermediate

    272: .seealso: DMNetworkGetSharedVertices()
    273: @*/
    274: PetscErrorCode DMNetworkSharedVertexGetInfo(DM dm,PetscInt v,PetscInt *gidx,PetscInt *n,const PetscInt **sv)
    275: {
    276:   DM_Network     *network = (DM_Network*)dm->data;
    277:   SVtx           *svtx = network->svtx;
    278:   PetscInt       i,gidx_tmp;

    280:   DMNetworkGetGlobalVertexIndex(dm,v,&gidx_tmp);
    281:   PetscTableFind(network->svtable,gidx_tmp+1,&i);

    284:   i--;
    285:   if (gidx) *gidx = gidx_tmp;
    286:   if (n)    *n    = svtx[i].n;
    287:   if (sv)   *sv   = svtx[i].sv;
    288:   return 0;
    289: }

    291: /*
    292:   VtxGetInfo - Get info of an input vertex=(net,idx)

    294:   Input Parameters:
    295: + Nsvtx - global num of shared vertices
    296: . svtx - array of shared vertices (global)
    297: - (net,idx) - subnet number and local index for a vertex

    299:   Output Parameters:
    300: + gidx - global index of (net,idx)
    301: . svtype - see petsc/private/dmnetworkimpl.h
    302: - svtx_idx - ordering in the svtx array
    303: */
    304: static inline PetscErrorCode VtxGetInfo(PetscInt Nsvtx,SVtx *svtx,PetscInt net,PetscInt idx,PetscInt *gidx,SVtxType *svtype,PetscInt *svtx_idx)
    305: {
    306:   PetscInt i,j,*svto,g_idx;
    307:   SVtxType vtype;

    309:   if (!Nsvtx) return 0;

    311:   g_idx = -1;
    312:   vtype = SVNONE;

    314:   for (i=0; i<Nsvtx; i++) {
    315:     if (net == svtx[i].sv[0] && idx == svtx[i].sv[1]) {
    316:       g_idx = svtx[i].gidx;
    317:       vtype = SVFROM;
    318:     } else { /* loop over svtx[i].n */
    319:       for (j=1; j<svtx[i].n; j++) {
    320:         svto = svtx[i].sv + 2*j;
    321:         if (net == svto[0] && idx == svto[1]) {
    322:           /* input vertex net.idx is a shared to_vertex, output its global index and its svtype */
    323:           g_idx = svtx[i].gidx; /* output gidx for to_vertex */
    324:           vtype = SVTO;
    325:         }
    326:       }
    327:     }
    328:     if (vtype != SVNONE) break;
    329:   }
    330:   if (gidx)     *gidx     = g_idx;
    331:   if (svtype)   *svtype   = vtype;
    332:   if (svtx_idx) *svtx_idx = i;
    333:   return 0;
    334: }

    336: /*
    337:   TableAddSVtx - Add a new shared vertice from sedgelist[k] to a ctable svta

    339:   Input:  network, sedgelist, k, svta
    340:   Output: svta, tdata, ta2sv
    341: */
    342: static inline PetscErrorCode TableAddSVtx(DM_Network *network,PetscInt *sedgelist,PetscInt k,PetscTable svta,PetscInt* tdata,PetscInt *ta2sv)
    343: {
    344:   PetscInt       net,idx,gidx;

    346:   net = sedgelist[k];
    347:   idx = sedgelist[k+1];
    348:   gidx = network->subnet[net].vStart + idx;
    349:   PetscTableAdd(svta,gidx+1,*tdata+1,INSERT_VALUES);

    351:   ta2sv[*tdata] = k; /* maps tdata to index of sedgelist */
    352:   (*tdata)++;
    353:   return 0;
    354: }

    356: /*
    357:   SharedVtxCreate - Create an array of global shared vertices. See SVtx and SVtxType in dmnetworkimpl.h

    359:   Input:  dm, Nsedgelist, sedgelist

    361:   Note: Output svtx is organized as
    362:         sv(net[0],idx[0]) --> sv(net[1],idx[1])
    363:                           --> sv(net[1],idx[1])
    364:                           ...
    365:                           --> sv(net[n-1],idx[n-1])
    366:         and net[0] < net[1] < ... < net[n-1]
    367:         where sv[0] has SVFROM type, sv[i], i>0, has SVTO type.
    368:  */
    369: static PetscErrorCode SharedVtxCreate(DM dm,PetscInt Nsedgelist,PetscInt *sedgelist)
    370: {
    371:   SVtx               *svtx = NULL;
    372:   PetscInt           *sv,k,j,nsv,*tdata,**ta2sv;
    373:   PetscTable         *svtas;
    374:   PetscInt           gidx,net,idx,i,nta,ita,idx_from,idx_to,n;
    375:   DM_Network         *network = (DM_Network*)dm->data;
    376:   PetscTablePosition ppos;

    378:   /* (1) Crete an array of ctables svtas to map (net,idx) -> gidx; a svtas[] for a shared/merged vertex */
    379:   PetscCalloc3(Nsedgelist,&svtas,Nsedgelist,&tdata,2*Nsedgelist,&ta2sv);

    381:   k   = 0;   /* sedgelist vertex counter j = 4*k */
    382:   nta = 0;   /* num of svta tables created */

    384:   /* for j=0 */
    385:   PetscTableCreate(2*Nsedgelist,network->NVertices+1,&svtas[nta]);
    386:   PetscMalloc1(2*Nsedgelist,&ta2sv[nta]);

    388:   TableAddSVtx(network,sedgelist,k,svtas[nta],&tdata[nta],ta2sv[nta]);
    389:   TableAddSVtx(network,sedgelist,k+2,svtas[nta],&tdata[nta],ta2sv[nta]);
    390:   nta++; k += 4;

    392:   for (j = 1; j < Nsedgelist; j++) { /* j: sedgelist counter */
    393:     for (ita = 0; ita < nta; ita++) {
    394:       /* vfrom */
    395:       net = sedgelist[k]; idx = sedgelist[k+1];
    396:       gidx = network->subnet[net].vStart + idx; /* global index of the vertex net.idx before merging shared vertices */
    397:       PetscTableFind(svtas[ita],gidx+1,&idx_from);

    399:       /* vto */
    400:       net = sedgelist[k+2]; idx = sedgelist[k+3];
    401:       gidx = network->subnet[net].vStart + idx;
    402:       PetscTableFind(svtas[ita],gidx+1,&idx_to);

    404:       if (idx_from || idx_to) { /* vfrom or vto is on table svtas[ita] */
    405:         idx_from--; idx_to--;
    406:         if (idx_from < 0) { /* vto is on svtas[ita] */
    407:           TableAddSVtx(network,sedgelist,k,svtas[ita],&tdata[ita],ta2sv[ita]);
    408:           break;
    409:         } else if (idx_to < 0) {
    410:           TableAddSVtx(network,sedgelist,k+2,svtas[ita],&tdata[ita],ta2sv[ita]);
    411:           break;
    412:         }
    413:       }
    414:     }

    416:     if (ita == nta) {
    417:       PetscTableCreate(2*Nsedgelist,network->NVertices+1,&svtas[nta]);
    418:       PetscMalloc1(2*Nsedgelist, &ta2sv[nta]);

    420:       TableAddSVtx(network,sedgelist,k,svtas[nta],&tdata[nta],ta2sv[nta]);
    421:       TableAddSVtx(network,sedgelist,k+2,svtas[nta],&tdata[nta],ta2sv[nta]);
    422:       nta++;
    423:     }
    424:     k += 4;
    425:   }

    427:   /* (2) Create svtable for querry shared vertices using gidx */
    428:   PetscTableCreate(nta,network->NVertices+1,&network->svtable);

    430:   /* (3) Construct svtx from svtas
    431:      svtx: array of SVtx: sv[0]=(net[0],idx[0]) to vertices sv[k], k=1,...,n-1;
    432:      net[k], k=0, ...,n-1, are in ascending order */
    433:   PetscMalloc1(nta,&svtx);
    434:   for (nsv = 0; nsv < nta; nsv++) {
    435:     /* for a single svtx, put shared vertices in ascending order of gidx */
    436:     PetscTableGetCount(svtas[nsv],&n);
    437:     PetscCalloc1(2*n,&sv);
    438:     svtx[nsv].sv   = sv;
    439:     svtx[nsv].n    = n;
    440:     svtx[nsv].gidx = network->NVertices; /* initialization */

    442:     PetscTableGetHeadPosition(svtas[nsv],&ppos);
    443:     for (k=0; k<n; k++) { /* gidx is sorted in ascending order */
    444:       PetscTableGetNext(svtas[nsv],&ppos,&gidx,&i);
    445:       gidx--; i--;

    447:       if (svtx[nsv].gidx > gidx) svtx[nsv].gidx = gidx; /*svtx[nsv].gidx = min(gidx) */

    449:       j = ta2sv[nsv][i]; /* maps i to index of sedgelist */
    450:       sv[2*k]   = sedgelist[j];   /* subnet number */
    451:       sv[2*k+1] = sedgelist[j+1]; /* index on the subnet */
    452:     }

    454:     /* Setup svtable for querry shared vertices */
    455:     PetscTableAdd(network->svtable,svtx[nsv].gidx+1,nsv+1,INSERT_VALUES);
    456:   }

    458:   for (j=0; j<nta; j++) {
    459:     PetscTableDestroy(&svtas[j]);
    460:     PetscFree(ta2sv[j]);
    461:   }
    462:   PetscFree3(svtas,tdata,ta2sv);

    464:   network->Nsvtx = nta;
    465:   network->svtx  = svtx;
    466:   return 0;
    467: }

    469: /*
    470:   GetEdgelist_Coupling - Get an integrated edgelist for dmplex from user-provided subnet[].edgelist when subnets are coupled by shared vertices

    472:   Input Parameters:
    473: . dm - the dmnetwork object

    475:    Output Parameters:
    476: +  edges - the integrated edgelist for dmplex
    477: -  nmerged_ptr - num of vertices being merged
    478: */
    479: static PetscErrorCode GetEdgelist_Coupling(DM dm,PetscInt *edges,PetscInt *nmerged_ptr)
    480: {
    481:   MPI_Comm       comm;
    482:   PetscMPIInt    size,rank,*recvcounts=NULL,*displs=NULL;
    483:   DM_Network     *network = (DM_Network*)dm->data;
    484:   PetscInt       i,j,ctr,np;
    485:   PetscInt       *vidxlTog,Nsv,Nsubnet=network->Nsubnet;
    486:   PetscInt       *sedgelist=network->sedgelist;
    487:   PetscInt       net,idx,gidx,nmerged,*vrange,gidx_from,net_from,sv_idx;
    488:   SVtxType       svtype = SVNONE;
    489:   SVtx           *svtx;

    491:   PetscObjectGetComm((PetscObject)dm,&comm);
    492:   MPI_Comm_rank(comm,&rank);
    493:   MPI_Comm_size(comm,&size);

    495:   /* (1) Create global svtx[] from sedgelist */
    496:   /* --------------------------------------- */
    497:   SharedVtxCreate(dm,network->Nsvtx,sedgelist);
    498:   Nsv  = network->Nsvtx;
    499:   svtx = network->svtx;

    501:   /* (2) Merge shared vto vertices to their vfrom vertex with same global vetex index (gidx) */
    502:   /* --------------------------------------------------------------------------------------- */
    503:   /* (2.1) compute vrage[rank]: global index of 1st local vertex in proc[rank] */
    504:   PetscMalloc4(size+1,&vrange,size,&displs,size,&recvcounts,network->nVertices,&vidxlTog);
    505:   for (i=0; i<size; i++) {displs[i] = i; recvcounts[i] = 1;}

    507:   vrange[0] = 0;
    508:   MPI_Allgatherv(&network->nVertices,1,MPIU_INT,vrange+1,recvcounts,displs,MPIU_INT,comm);
    509:   for (i=2; i<size+1; i++) vrange[i] += vrange[i-1];

    511:   /* (2.2) Create vidxlTog: maps UN-MERGED local vertex index i to global index gidx (plex, excluding ghost vertices) */
    512:   i = 0; gidx = 0;
    513:   nmerged        = 0; /* local num of merged vertices */
    514:   network->nsvtx = 0; /* local num of SVtx structs, including ghosts */
    515:   for (net=0; net<Nsubnet; net++) {
    516:     for (idx=0; idx<network->subnet[net].Nvtx; idx++) { /* Note: global subnet[net].Nvtx */
    517:       VtxGetInfo(Nsv,svtx,net,idx,&gidx_from,&svtype,&sv_idx);
    518:       if (svtype == SVTO) {
    519:         if (network->subnet[net].nvtx) {/* this proc owns sv_to */
    520:           net_from = svtx[sv_idx].sv[0]; /* subnet number of its shared vertex */
    521:           if (network->subnet[net_from].nvtx == 0) {
    522:             /* this proc does not own v_from, thus a ghost local vertex */
    523:             network->nsvtx++;
    524:           }
    525:           vidxlTog[i++] = gidx_from; /* gidx before merging! Bug??? */
    526:           nmerged++; /* a shared vertex -- merged */
    527:         }
    528:       } else {
    529:         if (svtype == SVFROM && network->subnet[net].nvtx) {
    530:           /* this proc owns this v_from, a new local shared vertex */
    531:           network->nsvtx++;
    532:         }
    533:         if (network->subnet[net].nvtx) vidxlTog[i++] = gidx;
    534:         gidx++;
    535:       }
    536:     }
    537:   }
    538: #if defined(PETSC_USE_DEBUG)
    540: #endif

    542:   /* (2.3) Shared vertices in the subnetworks are merged, update global NVertices: np = sum(local nmerged) */
    543:   MPI_Allreduce(&nmerged,&np,1,MPIU_INT,MPI_SUM,comm);
    544:   network->NVertices -= np;

    546:   ctr = 0;
    547:   for (net=0; net<Nsubnet; net++) {
    548:     for (j = 0; j < network->subnet[net].nedge; j++) {
    549:       /* vfrom: */
    550:       i = network->subnet[net].edgelist[2*j] + (network->subnet[net].vStart - vrange[rank]);
    551:       edges[2*ctr] = vidxlTog[i];

    553:       /* vto */
    554:       i = network->subnet[net].edgelist[2*j+1] + (network->subnet[net].vStart - vrange[rank]);
    555:       edges[2*ctr+1] = vidxlTog[i];
    556:       ctr++;
    557:     }
    558:   }
    559:   PetscFree4(vrange,displs,recvcounts,vidxlTog);
    560:   PetscFree(sedgelist); /* created in DMNetworkAddSharedVertices() */

    562:   *nmerged_ptr = nmerged;
    563:   return 0;
    564: }

    566: /*@
    567:   DMNetworkLayoutSetUp - Sets up the bare layout (graph) for the network

    569:   Not Collective

    571:   Input Parameters:
    572: . dm - the dmnetwork object

    574:   Notes:
    575:   This routine should be called after the network sizes and edgelists have been provided. It creates
    576:   the bare layout of the network and sets up the network to begin insertion of components.

    578:   All the components should be registered before calling this routine.

    580:   Level: beginner

    582: .seealso: DMNetworkSetNumSubNetworks(), DMNetworkAddSubnetwork()
    583: @*/
    584: PetscErrorCode DMNetworkLayoutSetUp(DM dm)
    585: {
    586:   DM_Network     *network = (DM_Network*)dm->data;
    587:   PetscInt       i,j,ctr,Nsubnet=network->Nsubnet,*eowners,np,*edges,*subnetvtx,*subnetedge,e,v,vfrom,vto,net;
    588:   const PetscInt *cone;
    589:   MPI_Comm       comm;
    590:   PetscMPIInt    size,rank;
    591:   PetscSection   sectiong;
    592:   PetscInt       nmerged=0;

    596:   /* This implementation requires user input each subnet by a single processor when Nsubnet>1, thus subnet[net].nvtx=subnet[net].Nvtx when net>0 */
    597:   for (net=1; net<Nsubnet; net++) {
    599:   }

    601:   PetscObjectGetComm((PetscObject)dm,&comm);
    602:   MPI_Comm_rank(comm,&rank);
    603:   MPI_Comm_size(comm,&size);

    605:   /* Create LOCAL edgelist in global vertex ordering for the network by concatenating local input edgelists of the subnetworks */
    606:   PetscCalloc2(2*network->nEdges,&edges,size+1,&eowners);

    608:   if (network->Nsvtx) { /* subnetworks are coupled via shared vertices */
    609:     GetEdgelist_Coupling(dm,edges,&nmerged);
    610:   } else { /* subnetworks are not coupled */
    611:     /* Create a 0-size svtable for querry shared vertices */
    612:     PetscTableCreate(0,network->NVertices+1,&network->svtable);
    613:     ctr = 0;
    614:     for (i=0; i < Nsubnet; i++) {
    615:       for (j = 0; j < network->subnet[i].nedge; j++) {
    616:         edges[2*ctr]   = network->subnet[i].vStart + network->subnet[i].edgelist[2*j];
    617:         edges[2*ctr+1] = network->subnet[i].vStart + network->subnet[i].edgelist[2*j+1];
    618:         ctr++;
    619:       }
    620:     }
    621:   }

    623:   /* Create network->plex; One dimensional network, numCorners=2 */
    624:   DMCreate(comm,&network->plex);
    625:   DMSetType(network->plex,DMPLEX);
    626:   DMSetDimension(network->plex,1);

    628:   if (size == 1) {
    629:     DMPlexBuildFromCellList(network->plex,network->nEdges,PETSC_DECIDE,2,edges);
    630:   } else {
    631:     DMPlexBuildFromCellListParallel(network->plex,network->nEdges,PETSC_DECIDE,PETSC_DECIDE,2,edges,NULL, NULL);
    632:   }

    634:   DMPlexGetChart(network->plex,&network->pStart,&network->pEnd);
    635:   DMPlexGetHeightStratum(network->plex,0,&network->eStart,&network->eEnd);
    636:   DMPlexGetHeightStratum(network->plex,1,&network->vStart,&network->vEnd);

    638:   PetscSectionCreate(comm,&network->DataSection);
    639:   PetscSectionCreate(comm,&network->DofSection);
    640:   PetscSectionSetChart(network->DataSection,network->pStart,network->pEnd);
    641:   PetscSectionSetChart(network->DofSection,network->pStart,network->pEnd);

    643:   np = network->pEnd - network->pStart;
    644:   PetscCalloc2(np,&network->header,np,&network->cvalue);
    645:   for (i=0; i < np; i++) {
    646:     network->header[i].maxcomps = 1;
    647:     SetUpNetworkHeaderComponentValue(dm,&network->header[i],&network->cvalue[i]);
    648:   }

    650:   /* Create edge and vertex arrays for the subnetworks
    651:      This implementation assumes that DMNetwork reads
    652:      (1) a single subnetwork in parallel; or
    653:      (2) n subnetworks using n processors, one subnetwork/processor.
    654:   */
    655:   PetscCalloc2(network->nEdges,&subnetedge,network->nVertices+network->nsvtx,&subnetvtx); /* Maps local edge/vertex to local subnetwork's edge/vertex */
    656:   network->subnetedge = subnetedge;
    657:   network->subnetvtx  = subnetvtx;
    658:   for (j=0; j < Nsubnet; j++) {
    659:     network->subnet[j].edges = subnetedge;
    660:     subnetedge              += network->subnet[j].nedge;

    662:     network->subnet[j].vertices = subnetvtx;
    663:     subnetvtx                  += network->subnet[j].nvtx;
    664:   }
    665:   network->svertices = subnetvtx;

    667:   /* Get edge ownership */
    668:   np = network->eEnd - network->eStart;
    669:   MPI_Allgather(&np,1,MPIU_INT,eowners+1,1,MPIU_INT,comm);
    670:   eowners[0] = 0;
    671:   for (i=2; i<=size; i++) eowners[i] += eowners[i-1];

    673:   /* Setup local edge and vertex arrays for subnetworks */
    674:   e = 0;
    675:   for (i=0; i < Nsubnet; i++) {
    676:     ctr = 0;
    677:     for (j = 0; j < network->subnet[i].nedge; j++) {
    678:       /* edge e */
    679:       network->header[e].index    = e + eowners[rank];   /* Global edge index */
    680:       network->header[e].subnetid = i;
    681:       network->subnet[i].edges[j] = e;

    683:       network->header[e].ndata           = 0;
    684:       network->header[e].offset[0]       = 0;
    685:       network->header[e].offsetvarrel[0] = 0;
    686:       PetscSectionAddDof(network->DataSection,e,network->header[e].hsize);

    688:       /* connected vertices */
    689:       DMPlexGetCone(network->plex,e,&cone);

    691:       /* vertex cone[0] */
    692:       v = cone[0];
    693:       network->header[v].index     = edges[2*e];  /* Global vertex index */
    694:       network->header[v].subnetid  = i;           /* Subnetwork id */
    695:       if (Nsubnet == 1) {
    696:         network->subnet[i].vertices[v - network->vStart] = v; /* user's subnet[].idx = petsc's v */
    697:       } else {
    698:         vfrom = network->subnet[i].edgelist[2*ctr];     /* =subnet[i].idx, Global index! */
    699:         network->subnet[i].vertices[vfrom] = v; /* user's subnet[].dix = petsc's v */
    700:       }

    702:       /* vertex cone[1] */
    703:       v = cone[1];
    704:       network->header[v].index    = edges[2*e+1];   /* Global vertex index */
    705:       network->header[v].subnetid = i;              /* Subnetwork id */
    706:       if (Nsubnet == 1) {
    707:         network->subnet[i].vertices[v - network->vStart] = v; /* user's subnet[].idx = petsc's v */
    708:       } else {
    709:         vto = network->subnet[i].edgelist[2*ctr+1];     /* =subnet[i].idx, Global index! */
    710:         network->subnet[i].vertices[vto] = v; /* user's subnet[].dix = petsc's v */
    711:       }

    713:       e++; ctr++;
    714:     }
    715:   }
    716:   PetscFree2(edges,eowners);

    718:   /* Set local vertex array for the subnetworks */
    719:   j = 0;
    720:   for (v = network->vStart; v < network->vEnd; v++) {
    721:     network->header[v].ndata           = 0;
    722:     network->header[v].offset[0]       = 0;
    723:     network->header[v].offsetvarrel[0] = 0;
    724:     PetscSectionAddDof(network->DataSection,v,network->header[e].hsize);

    726:     /* local shared vertex */
    727:     PetscTableFind(network->svtable,network->header[v].index+1,&i);
    728:     if (i) network->svertices[j++] = v;
    729:   }

    731:   /* Create a global section to be used by DMNetworkIsGhostVertex() which is a non-collective routine */
    732:   /* see snes_tutorials_network-ex1_4 */
    733:   DMGetGlobalSection(network->plex,&sectiong);
    734:   return 0;
    735: }

    737: /*@C
    738:   DMNetworkGetSubnetwork - Returns the information about a requested subnetwork

    740:   Not collective

    742:   Input Parameters:
    743: + dm - the DM object
    744: - netnum - the global index of the subnetwork

    746:   Output Parameters:
    747: + nv - number of vertices (local)
    748: . ne - number of edges (local)
    749: . vtx - local vertices of the subnetwork
    750: - edge - local edges of the subnetwork

    752:   Notes:
    753:     Cannot call this routine before DMNetworkLayoutSetup()

    755:     The local vertices returned on each rank are determined by DMNetwork. The user does not have any control over what vertices are local.

    757:   Level: intermediate

    759: .seealso: DMNetworkCreate(), DMNetworkAddSubnetwork(), DMNetworkLayoutSetUp()
    760: @*/
    761: PetscErrorCode DMNetworkGetSubnetwork(DM dm,PetscInt netnum,PetscInt *nv,PetscInt *ne,const PetscInt **vtx,const PetscInt **edge)
    762: {
    763:   DM_Network *network = (DM_Network*)dm->data;

    766:   if (nv) *nv     = network->subnet[netnum].nvtx;
    767:   if (ne) *ne     = network->subnet[netnum].nedge;
    768:   if (vtx) *vtx   = network->subnet[netnum].vertices;
    769:   if (edge) *edge = network->subnet[netnum].edges;
    770:   return 0;
    771: }

    773: /*@
    774:   DMNetworkAddSharedVertices - Add shared vertices that connect two given subnetworks

    776:   Collective on dm

    778:   Input Parameters:
    779: + dm - the dm object
    780: . anetnum - first subnetwork global numbering returned by DMNetworkAddSubnetwork()
    781: . bnetnum - second subnetwork global numbering returned by DMNetworkAddSubnetwork()
    782: . nsvtx - number of vertices that are shared by the two subnetworks
    783: . asvtx - vertex index in the first subnetwork
    784: - bsvtx - vertex index in the second subnetwork

    786:   Level: beginner

    788: .seealso: DMNetworkCreate(), DMNetworkAddSubnetwork(), DMNetworkGetSharedVertices()
    789: @*/
    790: PetscErrorCode DMNetworkAddSharedVertices(DM dm,PetscInt anetnum,PetscInt bnetnum,PetscInt nsvtx,PetscInt asvtx[],PetscInt bsvtx[])
    791: {
    792:   DM_Network     *network = (DM_Network*)dm->data;
    793:   PetscInt       i,nsubnet = network->Nsubnet,*sedgelist,Nsvtx=network->Nsvtx;

    797:   if (!Nsvtx) {
    798:     /* allocate network->sedgelist to hold at most 2*nsubnet pairs of shared vertices */
    799:     PetscMalloc1(2*4*nsubnet,&network->sedgelist);
    800:   }

    802:   sedgelist = network->sedgelist;
    803:   for (i=0; i<nsvtx; i++) {
    804:     sedgelist[4*Nsvtx]   = anetnum; sedgelist[4*Nsvtx+1] = asvtx[i];
    805:     sedgelist[4*Nsvtx+2] = bnetnum; sedgelist[4*Nsvtx+3] = bsvtx[i];
    806:     Nsvtx++;
    807:   }
    809:   network->Nsvtx = Nsvtx;
    810:   return 0;
    811: }

    813: /*@C
    814:   DMNetworkGetSharedVertices - Returns the info for the shared vertices

    816:   Not collective

    818:   Input Parameter:
    819: . dm - the DM object

    821:   Output Parameters:
    822: + nsv - number of local shared vertices
    823: - svtx - local shared vertices

    825:   Notes:
    826:   Cannot call this routine before DMNetworkLayoutSetup()

    828:   Level: intermediate

    830: .seealso: DMNetworkGetSubnetwork(), DMNetworkLayoutSetUp(), DMNetworkAddSharedVertices()
    831: @*/
    832: PetscErrorCode DMNetworkGetSharedVertices(DM dm,PetscInt *nsv,const PetscInt **svtx)
    833: {
    834:   DM_Network *net = (DM_Network*)dm->data;

    837:   if (nsv)  *nsv  = net->nsvtx;
    838:   if (svtx) *svtx = net->svertices;
    839:   return 0;
    840: }

    842: /*@C
    843:   DMNetworkRegisterComponent - Registers the network component

    845:   Logically collective on dm

    847:   Input Parameters:
    848: + dm - the network object
    849: . name - the component name
    850: - size - the storage size in bytes for this component data

    852:    Output Parameters:
    853: .  key - an integer key that defines the component

    855:    Notes
    856:    This routine should be called by all processors before calling DMNetworkLayoutSetup().

    858:    Level: beginner

    860: .seealso: DMNetworkCreate(), DMNetworkLayoutSetUp()
    861: @*/
    862: PetscErrorCode DMNetworkRegisterComponent(DM dm,const char *name,size_t size,PetscInt *key)
    863: {
    864:   DM_Network            *network = (DM_Network*) dm->data;
    865:   DMNetworkComponent    *component=NULL,*newcomponent=NULL;
    866:   PetscBool             flg=PETSC_FALSE;
    867:   PetscInt              i;

    869:   if (!network->component) {
    870:     PetscCalloc1(network->max_comps_registered,&network->component);
    871:   }

    873:   for (i=0; i < network->ncomponent; i++) {
    874:     PetscStrcmp(network->component[i].name,name,&flg);
    875:     if (flg) {
    876:       *key = i;
    877:       return 0;
    878:     }
    879:   }

    881:   if (network->ncomponent == network->max_comps_registered) {
    882:     /* Reached max allowed so resize component */
    883:     network->max_comps_registered += 2;
    884:     PetscCalloc1(network->max_comps_registered,&newcomponent);
    885:     /* Copy over the previous component info */
    886:     for (i=0; i < network->ncomponent; i++) {
    887:       PetscStrcpy(newcomponent[i].name,network->component[i].name);
    888:       newcomponent[i].size = network->component[i].size;
    889:     }
    890:     /* Free old one */
    891:     PetscFree(network->component);
    892:     /* Update pointer */
    893:     network->component = newcomponent;
    894:   }

    896:   component = &network->component[network->ncomponent];

    898:   PetscStrcpy(component->name,name);
    899:   component->size = size/sizeof(DMNetworkComponentGenericDataType);
    900:   *key = network->ncomponent;
    901:   network->ncomponent++;
    902:   return 0;
    903: }

    905: /*@
    906:   DMNetworkGetVertexRange - Get the bounds [start, end) for the local vertices

    908:   Not Collective

    910:   Input Parameter:
    911: . dm - the DMNetwork object

    913:   Output Parameters:
    914: + vStart - the first vertex point
    915: - vEnd - one beyond the last vertex point

    917:   Level: beginner

    919: .seealso: DMNetworkGetEdgeRange()
    920: @*/
    921: PetscErrorCode DMNetworkGetVertexRange(DM dm,PetscInt *vStart,PetscInt *vEnd)
    922: {
    923:   DM_Network *network = (DM_Network*)dm->data;

    925:   if (vStart) *vStart = network->vStart;
    926:   if (vEnd) *vEnd = network->vEnd;
    927:   return 0;
    928: }

    930: /*@
    931:   DMNetworkGetEdgeRange - Get the bounds [start, end) for the local edges

    933:   Not Collective

    935:   Input Parameter:
    936: . dm - the DMNetwork object

    938:   Output Parameters:
    939: + eStart - The first edge point
    940: - eEnd - One beyond the last edge point

    942:   Level: beginner

    944: .seealso: DMNetworkGetVertexRange()
    945: @*/
    946: PetscErrorCode DMNetworkGetEdgeRange(DM dm,PetscInt *eStart,PetscInt *eEnd)
    947: {
    948:   DM_Network *network = (DM_Network*)dm->data;

    951:   if (eStart) *eStart = network->eStart;
    952:   if (eEnd)   *eEnd   = network->eEnd;
    953:   return 0;
    954: }

    956: static PetscErrorCode DMNetworkGetIndex(DM dm,PetscInt p,PetscInt *index)
    957: {
    958:   DM_Network *network = (DM_Network*)dm->data;

    960:   if (network->header) {
    961:     *index = network->header[p].index;
    962:   } else {
    963:     PetscInt                 offsetp;
    964:     DMNetworkComponentHeader header;

    966:     PetscSectionGetOffset(network->DataSection,p,&offsetp);
    967:     header = (DMNetworkComponentHeader)(network->componentdataarray+offsetp);
    968:     *index = header->index;
    969:   }
    970:   return 0;
    971: }

    973: /*@
    974:   DMNetworkGetGlobalEdgeIndex - Get the global numbering for the edge on the network

    976:   Not Collective

    978:   Input Parameters:
    979: + dm - DMNetwork object
    980: - p - edge point

    982:   Output Parameters:
    983: . index - the global numbering for the edge

    985:   Level: intermediate

    987: .seealso: DMNetworkGetGlobalVertexIndex()
    988: @*/
    989: PetscErrorCode DMNetworkGetGlobalEdgeIndex(DM dm,PetscInt p,PetscInt *index)
    990: {
    991:   DMNetworkGetIndex(dm,p,index);
    992:   return 0;
    993: }

    995: /*@
    996:   DMNetworkGetGlobalVertexIndex - Get the global numbering for the vertex on the network

    998:   Not Collective

    1000:   Input Parameters:
    1001: + dm - DMNetwork object
    1002: - p  - vertex point

    1004:   Output Parameters:
    1005: . index - the global numbering for the vertex

    1007:   Level: intermediate

    1009: .seealso: DMNetworkGetGlobalEdgeIndex(), DMNetworkGetLocalVertexIndex()
    1010: @*/
    1011: PetscErrorCode DMNetworkGetGlobalVertexIndex(DM dm,PetscInt p,PetscInt *index)
    1012: {
    1013:   DMNetworkGetIndex(dm,p,index);
    1014:   return 0;
    1015: }

    1017: /*@
    1018:   DMNetworkGetNumComponents - Get the number of components at a vertex/edge

    1020:   Not Collective

    1022:   Input Parameters:
    1023: + dm - the DMNetwork object
    1024: - p - vertex/edge point

    1026:   Output Parameters:
    1027: . numcomponents - Number of components at the vertex/edge

    1029:   Level: beginner

    1031: .seealso: DMNetworkRegisterComponent(), DMNetworkAddComponent()
    1032: @*/
    1033: PetscErrorCode DMNetworkGetNumComponents(DM dm,PetscInt p,PetscInt *numcomponents)
    1034: {
    1035:   PetscInt       offset;
    1036:   DM_Network     *network = (DM_Network*)dm->data;

    1038:   PetscSectionGetOffset(network->DataSection,p,&offset);
    1039:   *numcomponents = ((DMNetworkComponentHeader)(network->componentdataarray+offset))->ndata;
    1040:   return 0;
    1041: }

    1043: /*@
    1044:   DMNetworkGetLocalVecOffset - Get the offset for accessing the variables associated with a component at the given vertex/edge from the local vector

    1046:   Not Collective

    1048:   Input Parameters:
    1049: + dm - the DMNetwork object
    1050: . p - the edge or vertex point
    1051: - compnum - component number; use ALL_COMPONENTS if no specific component is requested

    1053:   Output Parameters:
    1054: . offset - the local offset

    1056:   Notes:
    1057:     These offsets can be passed to MatSetValuesLocal() for matrices obtained with DMCreateMatrix().

    1059:     For vectors obtained with DMCreateLocalVector() the offsets can be used with VecSetValues().

    1061:     For vectors obtained with DMCreateLocalVector() and the array obtained with VecGetArray(vec,&array) you can access or set
    1062:     the vector values with array[offset].

    1064:     For vectors obtained with DMCreateGlobalVector() the offsets can be used with VecSetValuesLocal().

    1066:   Level: intermediate

    1068: .seealso: DMGetLocalVector(), DMNetworkGetComponent(), DMNetworkGetGlobalVecOffset(), DMCreateGlobalVector(), VecGetArray(), VecSetValuesLocal(), MatSetValuesLocal()
    1069: @*/
    1070: PetscErrorCode DMNetworkGetLocalVecOffset(DM dm,PetscInt p,PetscInt compnum,PetscInt *offset)
    1071: {
    1072:   DM_Network     *network = (DM_Network*)dm->data;
    1073:   PetscInt       offsetp,offsetd;
    1074:   DMNetworkComponentHeader header;

    1076:   PetscSectionGetOffset(network->plex->localSection,p,&offsetp);
    1077:   if (compnum == ALL_COMPONENTS) {
    1078:     *offset = offsetp;
    1079:     return 0;
    1080:   }

    1082:   PetscSectionGetOffset(network->DataSection,p,&offsetd);
    1083:   header = (DMNetworkComponentHeader)(network->componentdataarray+offsetd);
    1084:   *offset = offsetp + header->offsetvarrel[compnum];
    1085:   return 0;
    1086: }

    1088: /*@
    1089:   DMNetworkGetGlobalVecOffset - Get the global offset for accessing the variables associated with a component for the given vertex/edge from the global vector

    1091:   Not Collective

    1093:   Input Parameters:
    1094: + dm - the DMNetwork object
    1095: . p - the edge or vertex point
    1096: - compnum - component number; use ALL_COMPONENTS if no specific component is requested

    1098:   Output Parameters:
    1099: . offsetg - the global offset

    1101:   Notes:
    1102:     These offsets can be passed to MatSetValues() for matrices obtained with DMCreateMatrix().

    1104:     For vectors obtained with DMCreateGlobalVector() the offsets can be used with VecSetValues().

    1106:     For vectors obtained with DMCreateGlobalVector() and the array obtained with VecGetArray(vec,&array) you can access or set
    1107:     the vector values with array[offset - rstart] where restart is obtained with VecGetOwnershipRange(v,&rstart,NULL);

    1109:   Level: intermediate

    1111: .seealso: DMNetworkGetLocalVecOffset(), DMGetGlobalVector(), DMNetworkGetComponent(), DMCreateGlobalVector(), VecGetArray(), VecSetValues(), MatSetValues()
    1112: @*/
    1113: PetscErrorCode DMNetworkGetGlobalVecOffset(DM dm,PetscInt p,PetscInt compnum,PetscInt *offsetg)
    1114: {
    1115:   DM_Network               *network = (DM_Network*)dm->data;
    1116:   PetscInt                 offsetp,offsetd;
    1117:   DMNetworkComponentHeader header;

    1119:   PetscSectionGetOffset(network->plex->globalSection,p,&offsetp);
    1120:   if (offsetp < 0) offsetp = -(offsetp + 1); /* Convert to actual global offset for ghost vertex */

    1122:   if (compnum == ALL_COMPONENTS) {
    1123:     *offsetg = offsetp;
    1124:     return 0;
    1125:   }
    1126:   PetscSectionGetOffset(network->DataSection,p,&offsetd);
    1127:   header = (DMNetworkComponentHeader)(network->componentdataarray+offsetd);
    1128:   *offsetg = offsetp + header->offsetvarrel[compnum];
    1129:   return 0;
    1130: }

    1132: /*@
    1133:   DMNetworkGetEdgeOffset - Get the offset for accessing the variables associated with the given edge from the local subvector

    1135:   Not Collective

    1137:   Input Parameters:
    1138: + dm - the DMNetwork object
    1139: - p - the edge point

    1141:   Output Parameters:
    1142: . offset - the offset

    1144:   Level: intermediate

    1146: .seealso: DMNetworkGetLocalVecOffset(), DMGetLocalVector()
    1147: @*/
    1148: PetscErrorCode DMNetworkGetEdgeOffset(DM dm,PetscInt p,PetscInt *offset)
    1149: {
    1150:   DM_Network     *network = (DM_Network*)dm->data;

    1152:   PetscSectionGetOffset(network->edge.DofSection,p,offset);
    1153:   return 0;
    1154: }

    1156: /*@
    1157:   DMNetworkGetVertexOffset - Get the offset for accessing the variables associated with the given vertex from the local subvector

    1159:   Not Collective

    1161:   Input Parameters:
    1162: + dm - the DMNetwork object
    1163: - p - the vertex point

    1165:   Output Parameters:
    1166: . offset - the offset

    1168:   Level: intermediate

    1170: .seealso: DMNetworkGetEdgeOffset(), DMGetLocalVector()
    1171: @*/
    1172: PetscErrorCode DMNetworkGetVertexOffset(DM dm,PetscInt p,PetscInt *offset)
    1173: {
    1174:   DM_Network     *network = (DM_Network*)dm->data;

    1176:   p -= network->vStart;
    1177:   PetscSectionGetOffset(network->vertex.DofSection,p,offset);
    1178:   return 0;
    1179: }

    1181: /*@
    1182:   DMNetworkAddComponent - Adds a network component and number of variables at the given point (vertex/edge)

    1184:   Collective on dm

    1186:   Input Parameters:
    1187: + dm - the DMNetwork
    1188: . p - the vertex/edge point. These points are local indices provided by DMNetworkGetSubnetwork()
    1189: . componentkey - component key returned while registering the component with DMNetworkRegisterComponent()
    1190: . compvalue - pointer to the data structure for the component, or NULL if the component does not require data
    1191: - nvar - number of variables for the component at the vertex/edge point, zero if the component does not introduce any degrees of freedom at the point

    1193:   Notes:
    1194:     The owning rank and any other ranks that have this point as a ghost location must call this routine to add a component and number of variables in the same order at the given point.

    1196:     DMNetworkLayoutSetUp() must be called before this routine.

    1198:   Developer Notes:
    1199:      The requirement that all the ranks with access to a vertex (as owner or as ghost) add all the components comes from a limitation of the underlying implementation based on DMPLEX.
    1200:   Level: beginner

    1202: .seealso: DMNetworkGetComponent(), DMNetworkGetSubnetwork(), DMNetworkIsGhostVertex(), DMNetworkLayoutSetUp()
    1203: @*/
    1204: PetscErrorCode DMNetworkAddComponent(DM dm,PetscInt p,PetscInt componentkey,void* compvalue,PetscInt nvar)
    1205: {
    1206:   DM_Network               *network = (DM_Network*)dm->data;
    1207:   DMNetworkComponent       *component = &network->component[componentkey];
    1208:   DMNetworkComponentHeader header;
    1209:   DMNetworkComponentValue  cvalue;
    1210:   PetscInt                 compnum;
    1211:   PetscInt                 *compsize,*compkey,*compoffset,*compnvar,*compoffsetvarrel;
    1212:   void*                    *compdata;

    1216:   /* The owning rank and all ghost ranks add nvar */
    1217:   PetscSectionAddDof(network->DofSection,p,nvar);

    1219:   /* The owning rank and all ghost ranks add a component, including compvalue=NULL */
    1220:   header = &network->header[p];
    1221:   cvalue = &network->cvalue[p];
    1222:   if (header->ndata == header->maxcomps) {
    1223:     PetscInt additional_size;

    1225:     /* Reached limit so resize header component arrays */
    1226:     header->maxcomps += 2;

    1228:     /* Allocate arrays for component information and value */
    1229:     PetscCalloc5(header->maxcomps,&compsize,header->maxcomps,&compkey,header->maxcomps,&compoffset,header->maxcomps,&compnvar,header->maxcomps,&compoffsetvarrel);
    1230:     PetscMalloc1(header->maxcomps,&compdata);

    1232:     /* Recalculate header size */
    1233:     header->hsize = sizeof(struct _p_DMNetworkComponentHeader) + 5*header->maxcomps*sizeof(PetscInt);

    1235:     header->hsize /= sizeof(DMNetworkComponentGenericDataType);

    1237:     /* Copy over component info */
    1238:     PetscMemcpy(compsize,header->size,header->ndata*sizeof(PetscInt));
    1239:     PetscMemcpy(compkey,header->key,header->ndata*sizeof(PetscInt));
    1240:     PetscMemcpy(compoffset,header->offset,header->ndata*sizeof(PetscInt));
    1241:     PetscMemcpy(compnvar,header->nvar,header->ndata*sizeof(PetscInt));
    1242:     PetscMemcpy(compoffsetvarrel,header->offsetvarrel,header->ndata*sizeof(PetscInt));

    1244:     /* Copy over component data pointers */
    1245:     PetscMemcpy(compdata,cvalue->data,header->ndata*sizeof(void*));

    1247:     /* Free old arrays */
    1248:     PetscFree5(header->size,header->key,header->offset,header->nvar,header->offsetvarrel);
    1249:     PetscFree(cvalue->data);

    1251:     /* Update pointers */
    1252:     header->size         = compsize;
    1253:     header->key          = compkey;
    1254:     header->offset       = compoffset;
    1255:     header->nvar         = compnvar;
    1256:     header->offsetvarrel = compoffsetvarrel;

    1258:     cvalue->data = compdata;

    1260:     /* Update DataSection Dofs */
    1261:     /* The dofs for datasection point p equals sizeof the header (i.e. header->hsize) + sizes of the components added at point p. With the resizing of the header, we need to update the dofs for point p. Hence, we add the extra size added for the header */
    1262:     additional_size = (5*(header->maxcomps - header->ndata)*sizeof(PetscInt))/sizeof(DMNetworkComponentGenericDataType);
    1263:     PetscSectionAddDof(network->DataSection,p,additional_size);
    1264:   }
    1265:   header = &network->header[p];
    1266:   cvalue = &network->cvalue[p];

    1268:   compnum = header->ndata;

    1270:   header->size[compnum] = component->size;
    1271:   PetscSectionAddDof(network->DataSection,p,component->size);
    1272:   header->key[compnum] = componentkey;
    1273:   if (compnum != 0) header->offset[compnum] = header->offset[compnum-1] + header->size[compnum-1];
    1274:   cvalue->data[compnum] = (void*)compvalue;

    1276:   /* variables */
    1277:   header->nvar[compnum] += nvar;
    1278:   if (compnum != 0) header->offsetvarrel[compnum] = header->offsetvarrel[compnum-1] + header->nvar[compnum-1];

    1280:   header->ndata++;
    1281:   return 0;
    1282: }

    1284: /*@
    1285:   DMNetworkGetComponent - Gets the component key, the component data, and the number of variables at a given network point

    1287:   Not Collective

    1289:   Input Parameters:
    1290: + dm - the DMNetwork object
    1291: . p - vertex/edge point
    1292: - compnum - component number; use ALL_COMPONENTS if sum up all the components

    1294:   Output Parameters:
    1295: + compkey - the key obtained when registering the component (use NULL if not required)
    1296: . component - the component data (use NULL if not required)
    1297: - nvar  - number of variables (use NULL if not required)

    1299:   Level: beginner

    1301: .seealso: DMNetworkAddComponent(), DMNetworkGetNumComponents()
    1302: @*/
    1303: PetscErrorCode DMNetworkGetComponent(DM dm,PetscInt p,PetscInt compnum,PetscInt *compkey,void **component,PetscInt *nvar)
    1304: {
    1305:   DM_Network     *network = (DM_Network*)dm->data;
    1306:   PetscInt       offset = 0;
    1307:   DMNetworkComponentHeader header;

    1309:   if (compnum == ALL_COMPONENTS) {
    1310:     PetscSectionGetDof(network->DofSection,p,nvar);
    1311:     return 0;
    1312:   }

    1314:   PetscSectionGetOffset(network->DataSection,p,&offset);
    1315:   header = (DMNetworkComponentHeader)(network->componentdataarray+offset);

    1317:   if (compnum >= 0) {
    1318:     if (compkey) *compkey = header->key[compnum];
    1319:     if (component) {
    1320:       offset += header->hsize+header->offset[compnum];
    1321:       *component = network->componentdataarray+offset;
    1322:     }
    1323:   }

    1325:   if (nvar) *nvar = header->nvar[compnum];

    1327:   return 0;
    1328: }

    1330: /*
    1331:  Sets up the array that holds the data for all components and its associated section.
    1332:  It copies the data for all components in a contiguous array called componentdataarray. The component data is stored pointwise with an additional header (metadata) stored for each point. The header has metadata information such as number of components at each point, number of variables for each component, offsets for the components data, etc.
    1333: */
    1334: PetscErrorCode DMNetworkComponentSetUp(DM dm)
    1335: {
    1336:   DM_Network               *network = (DM_Network*)dm->data;
    1337:   PetscInt                 arr_size,p,offset,offsetp,ncomp,i,*headerarr;
    1338:   DMNetworkComponentHeader header;
    1339:   DMNetworkComponentValue  cvalue;
    1340:   DMNetworkComponentHeader headerinfo;
    1341:   DMNetworkComponentGenericDataType *componentdataarray;

    1343:   PetscSectionSetUp(network->DataSection);
    1344:   PetscSectionGetStorageSize(network->DataSection,&arr_size);
    1345:   /* arr_size+1 fixes pipeline test of opensolaris-misc for src/dm/tests/ex10.c -- Do not know why */
    1346:   PetscCalloc1(arr_size+1,&network->componentdataarray);

    1348:   componentdataarray = network->componentdataarray;
    1349:   for (p = network->pStart; p < network->pEnd; p++) {
    1350:     PetscSectionGetOffset(network->DataSection,p,&offsetp);
    1351:     /* Copy header */
    1352:     header = &network->header[p];
    1353:     headerinfo = (DMNetworkComponentHeader)(componentdataarray+offsetp);
    1354:     PetscMemcpy(headerinfo,header,sizeof(struct _p_DMNetworkComponentHeader));
    1355:     headerarr = (PetscInt*)(headerinfo+1);
    1356:     PetscMemcpy(headerarr,header->size,header->maxcomps*sizeof(PetscInt));
    1357:     headerarr += header->maxcomps;
    1358:     PetscMemcpy(headerarr,header->key,header->maxcomps*sizeof(PetscInt));
    1359:     headerarr += header->maxcomps;
    1360:     PetscMemcpy(headerarr,header->offset,header->maxcomps*sizeof(PetscInt));
    1361:     headerarr += header->maxcomps;
    1362:     PetscMemcpy(headerarr,header->nvar,header->maxcomps*sizeof(PetscInt));
    1363:     headerarr += header->maxcomps;
    1364:     PetscMemcpy(headerarr,header->offsetvarrel,header->maxcomps*sizeof(PetscInt));

    1366:     /* Copy data */
    1367:     cvalue = &network->cvalue[p];
    1368:     ncomp  = header->ndata;

    1370:     for (i = 0; i < ncomp; i++) {
    1371:       offset = offsetp + header->hsize + header->offset[i];
    1372:       PetscMemcpy(componentdataarray+offset,cvalue->data[i],header->size[i]*sizeof(DMNetworkComponentGenericDataType));
    1373:     }
    1374:   }
    1375:   return 0;
    1376: }

    1378: /* Sets up the section for dofs. This routine is called during DMSetUp() */
    1379: static PetscErrorCode DMNetworkVariablesSetUp(DM dm)
    1380: {
    1381:   DM_Network     *network = (DM_Network*)dm->data;

    1383:   PetscSectionSetUp(network->DofSection);
    1384:   return 0;
    1385: }

    1387: /* Get a subsection from a range of points */
    1388: static PetscErrorCode DMNetworkGetSubSection_private(PetscSection main,PetscInt pstart,PetscInt pend,PetscSection *subsection)
    1389: {
    1390:   PetscInt       i, nvar;

    1392:   PetscSectionCreate(PetscObjectComm((PetscObject)main), subsection);
    1393:   PetscSectionSetChart(*subsection, 0, pend - pstart);
    1394:   for (i = pstart; i < pend; i++) {
    1395:     PetscSectionGetDof(main,i,&nvar);
    1396:     PetscSectionSetDof(*subsection, i - pstart, nvar);
    1397:   }

    1399:   PetscSectionSetUp(*subsection);
    1400:   return 0;
    1401: }

    1403: /* Create a submap of points with a GlobalToLocal structure */
    1404: static PetscErrorCode DMNetworkSetSubMap_private(PetscInt pstart, PetscInt pend, ISLocalToGlobalMapping *map)
    1405: {
    1406:   PetscInt       i, *subpoints;

    1408:   /* Create index sets to map from "points" to "subpoints" */
    1409:   PetscMalloc1(pend - pstart, &subpoints);
    1410:   for (i = pstart; i < pend; i++) {
    1411:     subpoints[i - pstart] = i;
    1412:   }
    1413:   ISLocalToGlobalMappingCreate(PETSC_COMM_WORLD,1,pend-pstart,subpoints,PETSC_COPY_VALUES,map);
    1414:   PetscFree(subpoints);
    1415:   return 0;
    1416: }

    1418: /*@
    1419:   DMNetworkAssembleGraphStructures - Assembles vertex and edge data structures. Must be called after DMNetworkDistribute

    1421:   Collective on dm

    1423:   Input Parameters:
    1424: . dm - the DMNetworkObject

    1426:   Note: the routine will create alternative orderings for the vertices and edges. Assume global network points are:

    1428:   points = [0 1 2 3 4 5 6]

    1430:   where edges = [0,1,2,3] and vertices = [4,5,6]. The new orderings will be specific to the subset (i.e vertices = [0,1,2] <- [4,5,6]).

    1432:   With this new ordering a local PetscSection, global PetscSection and PetscSF will be created specific to the subset.

    1434:   Level: intermediate

    1436: @*/
    1437: PetscErrorCode DMNetworkAssembleGraphStructures(DM dm)
    1438: {
    1439:   MPI_Comm       comm;
    1440:   PetscMPIInt    size;
    1441:   DM_Network     *network = (DM_Network*)dm->data;

    1443:   PetscObjectGetComm((PetscObject)dm,&comm);
    1444:   MPI_Comm_size(comm, &size);

    1446:   /* Create maps for vertices and edges */
    1447:   DMNetworkSetSubMap_private(network->vStart,network->vEnd,&network->vertex.mapping);
    1448:   DMNetworkSetSubMap_private(network->eStart,network->eEnd,&network->edge.mapping);

    1450:   /* Create local sub-sections */
    1451:   DMNetworkGetSubSection_private(network->DofSection,network->vStart,network->vEnd,&network->vertex.DofSection);
    1452:   DMNetworkGetSubSection_private(network->DofSection,network->eStart,network->eEnd,&network->edge.DofSection);

    1454:   if (size > 1) {
    1455:     PetscSFGetSubSF(network->plex->sf, network->vertex.mapping, &network->vertex.sf);

    1457:     PetscSectionCreateGlobalSection(network->vertex.DofSection, network->vertex.sf, PETSC_FALSE, PETSC_FALSE, &network->vertex.GlobalDofSection);
    1458:     PetscSFGetSubSF(network->plex->sf, network->edge.mapping, &network->edge.sf);
    1459:     PetscSectionCreateGlobalSection(network->edge.DofSection, network->edge.sf, PETSC_FALSE, PETSC_FALSE, &network->edge.GlobalDofSection);
    1460:   } else {
    1461:     /* create structures for vertex */
    1462:     PetscSectionClone(network->vertex.DofSection,&network->vertex.GlobalDofSection);
    1463:     /* create structures for edge */
    1464:     PetscSectionClone(network->edge.DofSection,&network->edge.GlobalDofSection);
    1465:   }

    1467:   /* Add viewers */
    1468:   PetscObjectSetName((PetscObject)network->edge.GlobalDofSection,"Global edge dof section");
    1469:   PetscObjectSetName((PetscObject)network->vertex.GlobalDofSection,"Global vertex dof section");
    1470:   PetscSectionViewFromOptions(network->edge.GlobalDofSection, NULL, "-edge_global_section_view");
    1471:   PetscSectionViewFromOptions(network->vertex.GlobalDofSection, NULL, "-vertex_global_section_view");
    1472:   return 0;
    1473: }

    1475: /*
    1476:    Setup a lookup btable for the input v's owning subnetworks
    1477:    - add all owing subnetworks that connect to this v to the btable
    1478:      vertex_subnetid = supportingedge_subnetid
    1479: */
    1480: static inline PetscErrorCode SetSubnetIdLookupBT(DM dm,PetscInt v,PetscInt Nsubnet,PetscBT btable)
    1481: {
    1482:   PetscInt       e,nedges,offset;
    1483:   const PetscInt *edges;
    1484:   DM_Network     *newDMnetwork = (DM_Network*)dm->data;
    1485:   DMNetworkComponentHeader header;

    1487:   PetscBTMemzero(Nsubnet,btable);
    1488:   DMNetworkGetSupportingEdges(dm,v,&nedges,&edges);
    1489:   for (e=0; e<nedges; e++) {
    1490:     PetscSectionGetOffset(newDMnetwork->DataSection,edges[e],&offset);
    1491:     header = (DMNetworkComponentHeader)(newDMnetwork->componentdataarray+offset);
    1492:     PetscBTSet(btable,header->subnetid);
    1493:   }
    1494:   return 0;
    1495: }

    1497: /*@
    1498:   DMNetworkDistribute - Distributes the network and moves associated component data

    1500:   Collective

    1502:   Input Parameters:
    1503: + DM - the DMNetwork object
    1504: - overlap - the overlap of partitions, 0 is the default

    1506:   Options Database Key:
    1507: + -dmnetwork_view - Calls DMView() at the conclusion of DMSetUp()
    1508: - -dmnetwork_view_distributed - Calls DMView() at the conclusion of DMNetworkDistribute()

    1510:   Notes:
    1511:   Distributes the network with <overlap>-overlapping partitioning of the edges.

    1513:   Level: intermediate

    1515: .seealso: DMNetworkCreate()
    1516: @*/
    1517: PetscErrorCode DMNetworkDistribute(DM *dm,PetscInt overlap)
    1518: {
    1519:   MPI_Comm       comm;
    1520:   PetscMPIInt    size;
    1521:   DM_Network     *oldDMnetwork = (DM_Network*)((*dm)->data);
    1522:   DM_Network     *newDMnetwork;
    1523:   PetscSF        pointsf=NULL;
    1524:   DM             newDM;
    1525:   PetscInt       j,e,v,offset,*subnetvtx,*subnetedge,Nsubnet,gidx,svtx_idx,nv;
    1526:   PetscInt       net,*sv;
    1527:   PetscBT        btable;
    1528:   PetscPartitioner         part;
    1529:   DMNetworkComponentHeader header;

    1533:   PetscObjectGetComm((PetscObject)*dm,&comm);
    1534:   MPI_Comm_size(comm, &size);
    1535:   if (size == 1) return 0;

    1539:   /* This routine moves the component data to the appropriate processors. It makes use of the DataSection and the componentdataarray to move the component data to appropriate processors and returns a new DataSection and new componentdataarray. */
    1540:   DMNetworkCreate(PetscObjectComm((PetscObject)*dm),&newDM);
    1541:   newDMnetwork = (DM_Network*)newDM->data;
    1542:   newDMnetwork->max_comps_registered = oldDMnetwork->max_comps_registered;
    1543:   PetscMalloc1(newDMnetwork->max_comps_registered,&newDMnetwork->component);

    1545:   /* Enable runtime options for petscpartitioner */
    1546:   DMPlexGetPartitioner(oldDMnetwork->plex,&part);
    1547:   PetscPartitionerSetFromOptions(part);

    1549:   /* Distribute plex dm */
    1550:   DMPlexDistribute(oldDMnetwork->plex,overlap,&pointsf,&newDMnetwork->plex);

    1552:   /* Distribute dof section */
    1553:   PetscSectionCreate(comm,&newDMnetwork->DofSection);
    1554:   PetscSFDistributeSection(pointsf,oldDMnetwork->DofSection,NULL,newDMnetwork->DofSection);

    1556:   /* Distribute data and associated section */
    1557:   PetscSectionCreate(comm,&newDMnetwork->DataSection);
    1558:   DMPlexDistributeData(newDMnetwork->plex,pointsf,oldDMnetwork->DataSection,MPIU_INT,(void*)oldDMnetwork->componentdataarray,newDMnetwork->DataSection,(void**)&newDMnetwork->componentdataarray);

    1560:   PetscSectionGetChart(newDMnetwork->DataSection,&newDMnetwork->pStart,&newDMnetwork->pEnd);
    1561:   DMPlexGetHeightStratum(newDMnetwork->plex,0, &newDMnetwork->eStart,&newDMnetwork->eEnd);
    1562:   DMPlexGetHeightStratum(newDMnetwork->plex,1,&newDMnetwork->vStart,&newDMnetwork->vEnd);
    1563:   newDMnetwork->nEdges    = newDMnetwork->eEnd - newDMnetwork->eStart;
    1564:   newDMnetwork->nVertices = newDMnetwork->vEnd - newDMnetwork->vStart;
    1565:   newDMnetwork->NVertices = oldDMnetwork->NVertices;
    1566:   newDMnetwork->NEdges    = oldDMnetwork->NEdges;
    1567:   newDMnetwork->svtable   = oldDMnetwork->svtable; /* global table! */
    1568:   oldDMnetwork->svtable   = NULL;

    1570:   /* Set Dof section as the section for dm */
    1571:   DMSetLocalSection(newDMnetwork->plex,newDMnetwork->DofSection);
    1572:   DMGetGlobalSection(newDMnetwork->plex,&newDMnetwork->GlobalDofSection);

    1574:   /* Setup subnetwork info in the newDM */
    1575:   newDMnetwork->Nsubnet = oldDMnetwork->Nsubnet;
    1576:   newDMnetwork->Nsvtx   = oldDMnetwork->Nsvtx;
    1577:   oldDMnetwork->Nsvtx   = 0;
    1578:   newDMnetwork->svtx    = oldDMnetwork->svtx; /* global vertices! */
    1579:   oldDMnetwork->svtx    = NULL;
    1580:   PetscCalloc1(newDMnetwork->Nsubnet,&newDMnetwork->subnet);

    1582:   /* Copy over the global number of vertices and edges in each subnetwork.
    1583:      Note: these are calculated in DMNetworkLayoutSetUp()
    1584:   */
    1585:   Nsubnet = newDMnetwork->Nsubnet;
    1586:   for (j = 0; j < Nsubnet; j++) {
    1587:     newDMnetwork->subnet[j].Nvtx  = oldDMnetwork->subnet[j].Nvtx;
    1588:     newDMnetwork->subnet[j].Nedge = oldDMnetwork->subnet[j].Nedge;
    1589:   }

    1591:   /* Count local nedges for subnetworks */
    1592:   for (e = newDMnetwork->eStart; e < newDMnetwork->eEnd; e++) {
    1593:     PetscSectionGetOffset(newDMnetwork->DataSection,e,&offset);
    1594:     header = (DMNetworkComponentHeader)(newDMnetwork->componentdataarray+offset);

    1596:     /* Update pointers */
    1597:     header->size          = (PetscInt*)(header + 1);
    1598:     header->key           = header->size   + header->maxcomps;
    1599:     header->offset        = header->key    + header->maxcomps;
    1600:     header->nvar          = header->offset + header->maxcomps;
    1601:     header->offsetvarrel  = header->nvar   + header->maxcomps;

    1603:     newDMnetwork->subnet[header->subnetid].nedge++;
    1604:   }

    1606:   /* Setup a btable to keep track subnetworks owned by this process at a shared vertex */
    1607:   if (newDMnetwork->Nsvtx) {
    1608:     PetscBTCreate(Nsubnet,&btable);
    1609:   }

    1611:   /* Count local nvtx for subnetworks */
    1612:   for (v = newDMnetwork->vStart; v < newDMnetwork->vEnd; v++) {
    1613:     PetscSectionGetOffset(newDMnetwork->DataSection,v,&offset);
    1614:     header = (DMNetworkComponentHeader)(newDMnetwork->componentdataarray+offset);

    1616:     /* Update pointers */
    1617:     header->size          = (PetscInt*)(header + 1);
    1618:     header->key           = header->size   + header->maxcomps;
    1619:     header->offset        = header->key    + header->maxcomps;
    1620:     header->nvar          = header->offset + header->maxcomps;
    1621:     header->offsetvarrel  = header->nvar   + header->maxcomps;

    1623:     /* shared vertices: use gidx=header->index to check if v is a shared vertex */
    1624:     gidx = header->index;
    1625:     PetscTableFind(newDMnetwork->svtable,gidx+1,&svtx_idx);
    1626:     svtx_idx--;

    1628:     if (svtx_idx < 0) { /* not a shared vertex */
    1629:       newDMnetwork->subnet[header->subnetid].nvtx++;
    1630:     } else { /* a shared vertex belongs to more than one subnetworks, it is being counted by multiple subnets */
    1631:       /* Setup a lookup btable for this v's owning subnetworks */
    1632:       SetSubnetIdLookupBT(newDM,v,Nsubnet,btable);

    1634:       for (j=0; j<newDMnetwork->svtx[svtx_idx].n; j++) {
    1635:         sv  = newDMnetwork->svtx[svtx_idx].sv + 2*j;
    1636:         net = sv[0];
    1637:         if (PetscBTLookup(btable,net)) newDMnetwork->subnet[net].nvtx++; /* sv is on net owned by this proces */
    1638:       }
    1639:     }
    1640:   }

    1642:   /* Get total local nvtx for subnetworks */
    1643:   nv = 0;
    1644:   for (j=0; j<Nsubnet; j++) nv += newDMnetwork->subnet[j].nvtx;
    1645:   nv += newDMnetwork->Nsvtx;

    1647:   /* Now create the vertices and edge arrays for the subnetworks */
    1648:   PetscCalloc2(newDMnetwork->nEdges,&subnetedge,nv,&subnetvtx); /* Maps local vertex to local subnetwork's vertex */
    1649:   newDMnetwork->subnetedge = subnetedge;
    1650:   newDMnetwork->subnetvtx  = subnetvtx;
    1651:   for (j=0; j < newDMnetwork->Nsubnet; j++) {
    1652:     newDMnetwork->subnet[j].edges = subnetedge;
    1653:     subnetedge                   += newDMnetwork->subnet[j].nedge;

    1655:     newDMnetwork->subnet[j].vertices = subnetvtx;
    1656:     subnetvtx                       += newDMnetwork->subnet[j].nvtx;

    1658:     /* Temporarily setting nvtx and nedge to 0 so we can use them as counters in the below for loop. These get updated when the vertices and edges are added. */
    1659:     newDMnetwork->subnet[j].nvtx = newDMnetwork->subnet[j].nedge = 0;
    1660:   }
    1661:   newDMnetwork->svertices = subnetvtx;

    1663:   /* Set the edges and vertices in each subnetwork */
    1664:   for (e = newDMnetwork->eStart; e < newDMnetwork->eEnd; e++) {
    1665:     PetscSectionGetOffset(newDMnetwork->DataSection,e,&offset);
    1666:     header = (DMNetworkComponentHeader)(newDMnetwork->componentdataarray+offset);
    1667:     newDMnetwork->subnet[header->subnetid].edges[newDMnetwork->subnet[header->subnetid].nedge++] = e;
    1668:   }

    1670:   nv = 0;
    1671:   for (v = newDMnetwork->vStart; v < newDMnetwork->vEnd; v++) {
    1672:     PetscSectionGetOffset(newDMnetwork->DataSection,v,&offset);
    1673:     header = (DMNetworkComponentHeader)(newDMnetwork->componentdataarray+offset);

    1675:     /* coupling vertices: use gidx = header->index to check if v is a coupling vertex */
    1676:     PetscTableFind(newDMnetwork->svtable,header->index+1,&svtx_idx);
    1677:     svtx_idx--;
    1678:     if (svtx_idx < 0) {
    1679:       newDMnetwork->subnet[header->subnetid].vertices[newDMnetwork->subnet[header->subnetid].nvtx++] = v;
    1680:     } else { /* a shared vertex */
    1681:       newDMnetwork->svertices[nv++] = v;

    1683:       /* Setup a lookup btable for this v's owning subnetworks */
    1684:       SetSubnetIdLookupBT(newDM,v,Nsubnet,btable);

    1686:       for (j=0; j<newDMnetwork->svtx[svtx_idx].n; j++) {
    1687:         sv  = newDMnetwork->svtx[svtx_idx].sv + 2*j;
    1688:         net = sv[0];
    1689:         if (PetscBTLookup(btable,net))
    1690:           newDMnetwork->subnet[net].vertices[newDMnetwork->subnet[net].nvtx++] = v;
    1691:       }
    1692:     }
    1693:   }
    1694:   newDMnetwork->nsvtx = nv;   /* num of local shared vertices */

    1696:   newDM->setupcalled = (*dm)->setupcalled;
    1697:   newDMnetwork->distributecalled = PETSC_TRUE;

    1699:   /* Free spaces */
    1700:   PetscSFDestroy(&pointsf);
    1701:   DMDestroy(dm);
    1702:   if (newDMnetwork->Nsvtx) {
    1703:     PetscBTDestroy(&btable);
    1704:   }

    1706:   /* View distributed dmnetwork */
    1707:   DMViewFromOptions(newDM,NULL,"-dmnetwork_view_distributed");

    1709:   *dm  = newDM;
    1710:   return 0;
    1711: }

    1713: /*@C
    1714:   PetscSFGetSubSF - Returns an SF for a specific subset of points. Leaves are re-numbered to reflect the new ordering

    1716:  Collective

    1718:   Input Parameters:
    1719: + mainSF - the original SF structure
    1720: - map - a ISLocalToGlobal mapping that contains the subset of points

    1722:   Output Parameter:
    1723: . subSF - a subset of the mainSF for the desired subset.

    1725:   Level: intermediate
    1726: @*/
    1727: PetscErrorCode PetscSFGetSubSF(PetscSF mainsf,ISLocalToGlobalMapping map,PetscSF *subSF)
    1728: {
    1729:   PetscInt              nroots, nleaves, *ilocal_sub;
    1730:   PetscInt              i, *ilocal_map, nroots_sub, nleaves_sub = 0;
    1731:   PetscInt              *local_points, *remote_points;
    1732:   PetscSFNode           *iremote_sub;
    1733:   const PetscInt        *ilocal;
    1734:   const PetscSFNode     *iremote;

    1736:   PetscSFGetGraph(mainsf,&nroots,&nleaves,&ilocal,&iremote);

    1738:   /* Look for leaves that pertain to the subset of points. Get the local ordering */
    1739:   PetscMalloc1(nleaves,&ilocal_map);
    1740:   ISGlobalToLocalMappingApply(map,IS_GTOLM_MASK,nleaves,ilocal,NULL,ilocal_map);
    1741:   for (i = 0; i < nleaves; i++) {
    1742:     if (ilocal_map[i] != -1) nleaves_sub += 1;
    1743:   }
    1744:   /* Re-number ilocal with subset numbering. Need information from roots */
    1745:   PetscMalloc2(nroots,&local_points,nroots,&remote_points);
    1746:   for (i = 0; i < nroots; i++) local_points[i] = i;
    1747:   ISGlobalToLocalMappingApply(map,IS_GTOLM_MASK,nroots,local_points,NULL,local_points);
    1748:   PetscSFBcastBegin(mainsf, MPIU_INT, local_points, remote_points,MPI_REPLACE);
    1749:   PetscSFBcastEnd(mainsf, MPIU_INT, local_points, remote_points,MPI_REPLACE);
    1750:   /* Fill up graph using local (that is, local to the subset) numbering. */
    1751:   PetscMalloc1(nleaves_sub,&ilocal_sub);
    1752:   PetscMalloc1(nleaves_sub,&iremote_sub);
    1753:   nleaves_sub = 0;
    1754:   for (i = 0; i < nleaves; i++) {
    1755:     if (ilocal_map[i] != -1) {
    1756:       ilocal_sub[nleaves_sub] = ilocal_map[i];
    1757:       iremote_sub[nleaves_sub].rank = iremote[i].rank;
    1758:       iremote_sub[nleaves_sub].index = remote_points[ilocal[i]];
    1759:       nleaves_sub += 1;
    1760:     }
    1761:   }
    1762:   PetscFree2(local_points,remote_points);
    1763:   ISLocalToGlobalMappingGetSize(map,&nroots_sub);

    1765:   /* Create new subSF */
    1766:   PetscSFCreate(PETSC_COMM_WORLD,subSF);
    1767:   PetscSFSetFromOptions(*subSF);
    1768:   PetscSFSetGraph(*subSF,nroots_sub,nleaves_sub,ilocal_sub,PETSC_OWN_POINTER,iremote_sub,PETSC_COPY_VALUES);
    1769:   PetscFree(ilocal_map);
    1770:   PetscFree(iremote_sub);
    1771:   return 0;
    1772: }

    1774: /*@C
    1775:   DMNetworkGetSupportingEdges - Return the supporting edges for this vertex point

    1777:   Not Collective

    1779:   Input Parameters:
    1780: + dm - the DMNetwork object
    1781: - p  - the vertex point

    1783:   Output Parameters:
    1784: + nedges - number of edges connected to this vertex point
    1785: - edges  - list of edge points

    1787:   Level: beginner

    1789:   Fortran Notes:
    1790:   Since it returns an array, this routine is only available in Fortran 90, and you must
    1791:   include petsc.h90 in your code.

    1793: .seealso: DMNetworkCreate(), DMNetworkGetConnectedVertices()
    1794: @*/
    1795: PetscErrorCode DMNetworkGetSupportingEdges(DM dm,PetscInt vertex,PetscInt *nedges,const PetscInt *edges[])
    1796: {
    1797:   DM_Network     *network = (DM_Network*)dm->data;

    1799:   DMPlexGetSupportSize(network->plex,vertex,nedges);
    1800:   if (edges) DMPlexGetSupport(network->plex,vertex,edges);
    1801:   return 0;
    1802: }

    1804: /*@C
    1805:   DMNetworkGetConnectedVertices - Return the connected vertices for this edge point

    1807:   Not Collective

    1809:   Input Parameters:
    1810: + dm - the DMNetwork object
    1811: - p - the edge point

    1813:   Output Parameters:
    1814: . vertices - vertices connected to this edge

    1816:   Level: beginner

    1818:   Fortran Notes:
    1819:   Since it returns an array, this routine is only available in Fortran 90, and you must
    1820:   include petsc.h90 in your code.

    1822: .seealso: DMNetworkCreate(), DMNetworkGetSupportingEdges()
    1823: @*/
    1824: PetscErrorCode DMNetworkGetConnectedVertices(DM dm,PetscInt edge,const PetscInt *vertices[])
    1825: {
    1826:   DM_Network     *network = (DM_Network*)dm->data;

    1828:   DMPlexGetCone(network->plex,edge,vertices);
    1829:   return 0;
    1830: }

    1832: /*@
    1833:   DMNetworkIsSharedVertex - Returns TRUE if the vertex is shared by subnetworks

    1835:   Not Collective

    1837:   Input Parameters:
    1838: + dm - the DMNetwork object
    1839: - p - the vertex point

    1841:   Output Parameter:
    1842: . flag - TRUE if the vertex is shared by subnetworks

    1844:   Level: beginner

    1846: .seealso: DMNetworkAddSharedVertices(), DMNetworkIsGhostVertex()
    1847: @*/
    1848: PetscErrorCode DMNetworkIsSharedVertex(DM dm,PetscInt p,PetscBool *flag)
    1849: {
    1850:   PetscInt       i;

    1852:   *flag = PETSC_FALSE;

    1854:   if (dm->setupcalled) { /* DMNetworkGetGlobalVertexIndex() requires DMSetUp() be called */
    1855:     DM_Network     *network = (DM_Network*)dm->data;
    1856:     PetscInt       gidx;
    1857:     DMNetworkGetGlobalVertexIndex(dm,p,&gidx);
    1858:     PetscTableFind(network->svtable,gidx+1,&i);
    1859:     if (i) *flag = PETSC_TRUE;
    1860:   } else { /* would be removed? */
    1861:     PetscInt       nv;
    1862:     const PetscInt *vtx;
    1863:     DMNetworkGetSharedVertices(dm,&nv,&vtx);
    1864:     for (i=0; i<nv; i++) {
    1865:       if (p == vtx[i]) {
    1866:         *flag = PETSC_TRUE;
    1867:         break;
    1868:       }
    1869:     }
    1870:   }
    1871:   return 0;
    1872: }

    1874: /*@
    1875:   DMNetworkIsGhostVertex - Returns TRUE if the vertex is a ghost vertex

    1877:   Not Collective

    1879:   Input Parameters:
    1880: + dm - the DMNetwork object
    1881: - p - the vertex point

    1883:   Output Parameter:
    1884: . isghost - TRUE if the vertex is a ghost point

    1886:   Level: beginner

    1888: .seealso: DMNetworkGetConnectedVertices(), DMNetworkGetVertexRange(), DMNetworkIsSharedVertex()
    1889: @*/
    1890: PetscErrorCode DMNetworkIsGhostVertex(DM dm,PetscInt p,PetscBool *isghost)
    1891: {
    1892:   DM_Network     *network = (DM_Network*)dm->data;
    1893:   PetscInt       offsetg;
    1894:   PetscSection   sectiong;

    1896:   *isghost = PETSC_FALSE;
    1897:   DMGetGlobalSection(network->plex,&sectiong);
    1898:   PetscSectionGetOffset(sectiong,p,&offsetg);
    1899:   if (offsetg < 0) *isghost = PETSC_TRUE;
    1900:   return 0;
    1901: }

    1903: PetscErrorCode DMSetUp_Network(DM dm)
    1904: {
    1905:   DM_Network     *network=(DM_Network*)dm->data;

    1907:   DMNetworkComponentSetUp(dm);
    1908:   DMNetworkVariablesSetUp(dm);

    1910:   DMSetLocalSection(network->plex,network->DofSection);
    1911:   DMGetGlobalSection(network->plex,&network->GlobalDofSection);

    1913:   dm->setupcalled = PETSC_TRUE;

    1915:   /* View dmnetwork */
    1916:   DMViewFromOptions(dm,NULL,"-dmnetwork_view");
    1917:   return 0;
    1918: }

    1920: /*@
    1921:   DMNetworkHasJacobian - Sets global flag for using user's sub Jacobian matrices
    1922:       -- replaced by DMNetworkSetOption(network,userjacobian,PETSC_TURE)?

    1924:   Collective

    1926:   Input Parameters:
    1927: + dm - the DMNetwork object
    1928: . eflg - turn the option on (PETSC_TRUE) or off (PETSC_FALSE) if user provides Jacobian for edges
    1929: - vflg - turn the option on (PETSC_TRUE) or off (PETSC_FALSE) if user provides Jacobian for vertices

    1931:  Level: intermediate

    1933: @*/
    1934: PetscErrorCode DMNetworkHasJacobian(DM dm,PetscBool eflg,PetscBool vflg)
    1935: {
    1936:   DM_Network     *network=(DM_Network*)dm->data;
    1937:   PetscInt       nVertices = network->nVertices;

    1939:   network->userEdgeJacobian   = eflg;
    1940:   network->userVertexJacobian = vflg;

    1942:   if (eflg && !network->Je) {
    1943:     PetscCalloc1(3*network->nEdges,&network->Je);
    1944:   }

    1946:   if (vflg && !network->Jv && nVertices) {
    1947:     PetscInt       i,*vptr,nedges,vStart=network->vStart;
    1948:     PetscInt       nedges_total;
    1949:     const PetscInt *edges;

    1951:     /* count nvertex_total */
    1952:     nedges_total = 0;
    1953:     PetscMalloc1(nVertices+1,&vptr);

    1955:     vptr[0] = 0;
    1956:     for (i=0; i<nVertices; i++) {
    1957:       DMNetworkGetSupportingEdges(dm,i+vStart,&nedges,&edges);
    1958:       nedges_total += nedges;
    1959:       vptr[i+1] = vptr[i] + 2*nedges + 1;
    1960:     }

    1962:     PetscCalloc1(2*nedges_total+nVertices,&network->Jv);
    1963:     network->Jvptr = vptr;
    1964:   }
    1965:   return 0;
    1966: }

    1968: /*@
    1969:   DMNetworkEdgeSetMatrix - Sets user-provided Jacobian matrices for this edge to the network

    1971:   Not Collective

    1973:   Input Parameters:
    1974: + dm - the DMNetwork object
    1975: . p - the edge point
    1976: - J - array (size = 3) of Jacobian submatrices for this edge point:
    1977:         J[0]: this edge
    1978:         J[1] and J[2]: connected vertices, obtained by calling DMNetworkGetConnectedVertices()

    1980:   Level: advanced

    1982: .seealso: DMNetworkVertexSetMatrix()
    1983: @*/
    1984: PetscErrorCode DMNetworkEdgeSetMatrix(DM dm,PetscInt p,Mat J[])
    1985: {
    1986:   DM_Network *network=(DM_Network*)dm->data;

    1990:   if (J) {
    1991:     network->Je[3*p]   = J[0];
    1992:     network->Je[3*p+1] = J[1];
    1993:     network->Je[3*p+2] = J[2];
    1994:   }
    1995:   return 0;
    1996: }

    1998: /*@
    1999:   DMNetworkVertexSetMatrix - Sets user-provided Jacobian matrix for this vertex to the network

    2001:   Not Collective

    2003:   Input Parameters:
    2004: + dm - The DMNetwork object
    2005: . p - the vertex point
    2006: - J - array of Jacobian (size = 2*(num of supporting edges) + 1) submatrices for this vertex point:
    2007:         J[0]:       this vertex
    2008:         J[1+2*i]:   i-th supporting edge
    2009:         J[1+2*i+1]: i-th connected vertex

    2011:   Level: advanced

    2013: .seealso: DMNetworkEdgeSetMatrix()
    2014: @*/
    2015: PetscErrorCode DMNetworkVertexSetMatrix(DM dm,PetscInt p,Mat J[])
    2016: {
    2017:   DM_Network     *network=(DM_Network*)dm->data;
    2018:   PetscInt       i,*vptr,nedges,vStart=network->vStart;
    2019:   const PetscInt *edges;

    2023:   if (J) {
    2024:     vptr = network->Jvptr;
    2025:     network->Jv[vptr[p-vStart]] = J[0]; /* Set Jacobian for this vertex */

    2027:     /* Set Jacobian for each supporting edge and connected vertex */
    2028:     DMNetworkGetSupportingEdges(dm,p,&nedges,&edges);
    2029:     for (i=1; i<=2*nedges; i++) network->Jv[vptr[p-vStart]+i] = J[i];
    2030:   }
    2031:   return 0;
    2032: }

    2034: static inline PetscErrorCode MatSetPreallocationDenseblock_private(PetscInt nrows,PetscInt *rows,PetscInt ncols,PetscBool ghost,Vec vdnz,Vec vonz)
    2035: {
    2036:   PetscInt       j;
    2037:   PetscScalar    val=(PetscScalar)ncols;

    2039:   if (!ghost) {
    2040:     for (j=0; j<nrows; j++) {
    2041:       VecSetValues(vdnz,1,&rows[j],&val,ADD_VALUES);
    2042:     }
    2043:   } else {
    2044:     for (j=0; j<nrows; j++) {
    2045:       VecSetValues(vonz,1,&rows[j],&val,ADD_VALUES);
    2046:     }
    2047:   }
    2048:   return 0;
    2049: }

    2051: static inline PetscErrorCode MatSetPreallocationUserblock_private(Mat Ju,PetscInt nrows,PetscInt *rows,PetscInt ncols,PetscBool ghost,Vec vdnz,Vec vonz)
    2052: {
    2053:   PetscInt       j,ncols_u;
    2054:   PetscScalar    val;

    2056:   if (!ghost) {
    2057:     for (j=0; j<nrows; j++) {
    2058:       MatGetRow(Ju,j,&ncols_u,NULL,NULL);
    2059:       val = (PetscScalar)ncols_u;
    2060:       VecSetValues(vdnz,1,&rows[j],&val,ADD_VALUES);
    2061:       MatRestoreRow(Ju,j,&ncols_u,NULL,NULL);
    2062:     }
    2063:   } else {
    2064:     for (j=0; j<nrows; j++) {
    2065:       MatGetRow(Ju,j,&ncols_u,NULL,NULL);
    2066:       val = (PetscScalar)ncols_u;
    2067:       VecSetValues(vonz,1,&rows[j],&val,ADD_VALUES);
    2068:       MatRestoreRow(Ju,j,&ncols_u,NULL,NULL);
    2069:     }
    2070:   }
    2071:   return 0;
    2072: }

    2074: static inline PetscErrorCode MatSetPreallocationblock_private(Mat Ju,PetscInt nrows,PetscInt *rows,PetscInt ncols,PetscBool ghost,Vec vdnz,Vec vonz)
    2075: {
    2076:   if (Ju) {
    2077:     MatSetPreallocationUserblock_private(Ju,nrows,rows,ncols,ghost,vdnz,vonz);
    2078:   } else {
    2079:     MatSetPreallocationDenseblock_private(nrows,rows,ncols,ghost,vdnz,vonz);
    2080:   }
    2081:   return 0;
    2082: }

    2084: static inline PetscErrorCode MatSetDenseblock_private(PetscInt nrows,PetscInt *rows,PetscInt ncols,PetscInt cstart,Mat *J)
    2085: {
    2086:   PetscInt       j,*cols;
    2087:   PetscScalar    *zeros;

    2089:   PetscCalloc2(ncols,&cols,nrows*ncols,&zeros);
    2090:   for (j=0; j<ncols; j++) cols[j] = j+ cstart;
    2091:   MatSetValues(*J,nrows,rows,ncols,cols,zeros,INSERT_VALUES);
    2092:   PetscFree2(cols,zeros);
    2093:   return 0;
    2094: }

    2096: static inline PetscErrorCode MatSetUserblock_private(Mat Ju,PetscInt nrows,PetscInt *rows,PetscInt ncols,PetscInt cstart,Mat *J)
    2097: {
    2098:   PetscInt       j,M,N,row,col,ncols_u;
    2099:   const PetscInt *cols;
    2100:   PetscScalar    zero=0.0;

    2102:   MatGetSize(Ju,&M,&N);

    2105:   for (row=0; row<nrows; row++) {
    2106:     MatGetRow(Ju,row,&ncols_u,&cols,NULL);
    2107:     for (j=0; j<ncols_u; j++) {
    2108:       col = cols[j] + cstart;
    2109:       MatSetValues(*J,1,&rows[row],1,&col,&zero,INSERT_VALUES);
    2110:     }
    2111:     MatRestoreRow(Ju,row,&ncols_u,&cols,NULL);
    2112:   }
    2113:   return 0;
    2114: }

    2116: static inline PetscErrorCode MatSetblock_private(Mat Ju,PetscInt nrows,PetscInt *rows,PetscInt ncols,PetscInt cstart,Mat *J)
    2117: {
    2118:   if (Ju) {
    2119:     MatSetUserblock_private(Ju,nrows,rows,ncols,cstart,J);
    2120:   } else {
    2121:     MatSetDenseblock_private(nrows,rows,ncols,cstart,J);
    2122:   }
    2123:   return 0;
    2124: }

    2126: /* Creates a GlobalToLocal mapping with a Local and Global section. This is akin to the routine DMGetLocalToGlobalMapping but without the need of providing a dm.
    2127: */
    2128: PetscErrorCode CreateSubGlobalToLocalMapping_private(PetscSection globalsec, PetscSection localsec, ISLocalToGlobalMapping *ltog)
    2129: {
    2130:   PetscInt       i,size,dof;
    2131:   PetscInt       *glob2loc;

    2133:   PetscSectionGetStorageSize(localsec,&size);
    2134:   PetscMalloc1(size,&glob2loc);

    2136:   for (i = 0; i < size; i++) {
    2137:     PetscSectionGetOffset(globalsec,i,&dof);
    2138:     dof = (dof >= 0) ? dof : -(dof + 1);
    2139:     glob2loc[i] = dof;
    2140:   }

    2142:   ISLocalToGlobalMappingCreate(PETSC_COMM_WORLD,1,size,glob2loc,PETSC_OWN_POINTER,ltog);
    2143: #if 0
    2144:   PetscIntView(size,glob2loc,PETSC_VIEWER_STDOUT_WORLD);
    2145: #endif
    2146:   return 0;
    2147: }

    2149: #include <petsc/private/matimpl.h>

    2151: PetscErrorCode DMCreateMatrix_Network_Nest(DM dm,Mat *J)
    2152: {
    2153:   DM_Network     *network = (DM_Network*)dm->data;
    2154:   PetscInt       eDof,vDof;
    2155:   Mat            j11,j12,j21,j22,bA[2][2];
    2156:   MPI_Comm       comm;
    2157:   ISLocalToGlobalMapping eISMap,vISMap;

    2159:   PetscObjectGetComm((PetscObject)dm,&comm);

    2161:   PetscSectionGetConstrainedStorageSize(network->edge.GlobalDofSection,&eDof);
    2162:   PetscSectionGetConstrainedStorageSize(network->vertex.GlobalDofSection,&vDof);

    2164:   MatCreate(comm, &j11);
    2165:   MatSetSizes(j11, eDof, eDof, PETSC_DETERMINE, PETSC_DETERMINE);
    2166:   MatSetType(j11, MATMPIAIJ);

    2168:   MatCreate(comm, &j12);
    2169:   MatSetSizes(j12, eDof, vDof, PETSC_DETERMINE ,PETSC_DETERMINE);
    2170:   MatSetType(j12, MATMPIAIJ);

    2172:   MatCreate(comm, &j21);
    2173:   MatSetSizes(j21, vDof, eDof, PETSC_DETERMINE, PETSC_DETERMINE);
    2174:   MatSetType(j21, MATMPIAIJ);

    2176:   MatCreate(comm, &j22);
    2177:   MatSetSizes(j22, vDof, vDof, PETSC_DETERMINE, PETSC_DETERMINE);
    2178:   MatSetType(j22, MATMPIAIJ);

    2180:   bA[0][0] = j11;
    2181:   bA[0][1] = j12;
    2182:   bA[1][0] = j21;
    2183:   bA[1][1] = j22;

    2185:   CreateSubGlobalToLocalMapping_private(network->edge.GlobalDofSection,network->edge.DofSection,&eISMap);
    2186:   CreateSubGlobalToLocalMapping_private(network->vertex.GlobalDofSection,network->vertex.DofSection,&vISMap);

    2188:   MatSetLocalToGlobalMapping(j11,eISMap,eISMap);
    2189:   MatSetLocalToGlobalMapping(j12,eISMap,vISMap);
    2190:   MatSetLocalToGlobalMapping(j21,vISMap,eISMap);
    2191:   MatSetLocalToGlobalMapping(j22,vISMap,vISMap);

    2193:   MatSetUp(j11);
    2194:   MatSetUp(j12);
    2195:   MatSetUp(j21);
    2196:   MatSetUp(j22);

    2198:   MatCreateNest(comm,2,NULL,2,NULL,&bA[0][0],J);
    2199:   MatSetUp(*J);
    2200:   MatNestSetVecType(*J,VECNEST);
    2201:   MatDestroy(&j11);
    2202:   MatDestroy(&j12);
    2203:   MatDestroy(&j21);
    2204:   MatDestroy(&j22);

    2206:   MatAssemblyBegin(*J,MAT_FINAL_ASSEMBLY);
    2207:   MatAssemblyEnd(*J,MAT_FINAL_ASSEMBLY);
    2208:   MatSetOption(*J,MAT_NEW_NONZERO_ALLOCATION_ERR,PETSC_TRUE);

    2210:   /* Free structures */
    2211:   ISLocalToGlobalMappingDestroy(&eISMap);
    2212:   ISLocalToGlobalMappingDestroy(&vISMap);
    2213:   return 0;
    2214: }

    2216: PetscErrorCode DMCreateMatrix_Network(DM dm,Mat *J)
    2217: {
    2218:   DM_Network     *network = (DM_Network*)dm->data;
    2219:   PetscInt       eStart,eEnd,vStart,vEnd,rstart,nrows,*rows,localSize;
    2220:   PetscInt       cstart,ncols,j,e,v;
    2221:   PetscBool      ghost,ghost_vc,ghost2,isNest;
    2222:   Mat            Juser;
    2223:   PetscSection   sectionGlobal;
    2224:   PetscInt       nedges,*vptr=NULL,vc,*rows_v; /* suppress maybe-uninitialized warning */
    2225:   const PetscInt *edges,*cone;
    2226:   MPI_Comm       comm;
    2227:   MatType        mtype;
    2228:   Vec            vd_nz,vo_nz;
    2229:   PetscInt       *dnnz,*onnz;
    2230:   PetscScalar    *vdnz,*vonz;

    2232:   mtype = dm->mattype;
    2233:   PetscStrcmp(mtype,MATNEST,&isNest);
    2234:   if (isNest) {
    2235:     DMCreateMatrix_Network_Nest(dm,J);
    2236:     MatSetDM(*J,dm);
    2237:     return 0;
    2238:   }

    2240:   if (!network->userEdgeJacobian && !network->userVertexJacobian) {
    2241:     /* user does not provide Jacobian blocks */
    2242:     DMCreateMatrix_Plex(network->plex,J);
    2243:     MatSetDM(*J,dm);
    2244:     return 0;
    2245:   }

    2247:   MatCreate(PetscObjectComm((PetscObject)dm),J);
    2248:   DMGetGlobalSection(network->plex,&sectionGlobal);
    2249:   PetscSectionGetConstrainedStorageSize(sectionGlobal,&localSize);
    2250:   MatSetSizes(*J,localSize,localSize,PETSC_DETERMINE,PETSC_DETERMINE);

    2252:   MatSetType(*J,MATAIJ);
    2253:   MatSetFromOptions(*J);

    2255:   /* (1) Set matrix preallocation */
    2256:   /*------------------------------*/
    2257:   PetscObjectGetComm((PetscObject)dm,&comm);
    2258:   VecCreate(comm,&vd_nz);
    2259:   VecSetSizes(vd_nz,localSize,PETSC_DECIDE);
    2260:   VecSetFromOptions(vd_nz);
    2261:   VecSet(vd_nz,0.0);
    2262:   VecDuplicate(vd_nz,&vo_nz);

    2264:   /* Set preallocation for edges */
    2265:   /*-----------------------------*/
    2266:   DMNetworkGetEdgeRange(dm,&eStart,&eEnd);

    2268:   PetscMalloc1(localSize,&rows);
    2269:   for (e=eStart; e<eEnd; e++) {
    2270:     /* Get row indices */
    2271:     DMNetworkGetGlobalVecOffset(dm,e,ALL_COMPONENTS,&rstart);
    2272:     PetscSectionGetDof(network->DofSection,e,&nrows);
    2273:     if (nrows) {
    2274:       for (j=0; j<nrows; j++) rows[j] = j + rstart;

    2276:       /* Set preallocation for connected vertices */
    2277:       DMNetworkGetConnectedVertices(dm,e,&cone);
    2278:       for (v=0; v<2; v++) {
    2279:         PetscSectionGetDof(network->DofSection,cone[v],&ncols);

    2281:         if (network->Je) {
    2282:           Juser = network->Je[3*e+1+v]; /* Jacobian(e,v) */
    2283:         } else Juser = NULL;
    2284:         DMNetworkIsGhostVertex(dm,cone[v],&ghost);
    2285:         MatSetPreallocationblock_private(Juser,nrows,rows,ncols,ghost,vd_nz,vo_nz);
    2286:       }

    2288:       /* Set preallocation for edge self */
    2289:       cstart = rstart;
    2290:       if (network->Je) {
    2291:         Juser = network->Je[3*e]; /* Jacobian(e,e) */
    2292:       } else Juser = NULL;
    2293:       MatSetPreallocationblock_private(Juser,nrows,rows,nrows,PETSC_FALSE,vd_nz,vo_nz);
    2294:     }
    2295:   }

    2297:   /* Set preallocation for vertices */
    2298:   /*--------------------------------*/
    2299:   DMNetworkGetVertexRange(dm,&vStart,&vEnd);
    2300:   if (vEnd - vStart) vptr = network->Jvptr;

    2302:   for (v=vStart; v<vEnd; v++) {
    2303:     /* Get row indices */
    2304:     DMNetworkGetGlobalVecOffset(dm,v,ALL_COMPONENTS,&rstart);
    2305:     PetscSectionGetDof(network->DofSection,v,&nrows);
    2306:     if (!nrows) continue;

    2308:     DMNetworkIsGhostVertex(dm,v,&ghost);
    2309:     if (ghost) {
    2310:       PetscMalloc1(nrows,&rows_v);
    2311:     } else {
    2312:       rows_v = rows;
    2313:     }

    2315:     for (j=0; j<nrows; j++) rows_v[j] = j + rstart;

    2317:     /* Get supporting edges and connected vertices */
    2318:     DMNetworkGetSupportingEdges(dm,v,&nedges,&edges);

    2320:     for (e=0; e<nedges; e++) {
    2321:       /* Supporting edges */
    2322:       DMNetworkGetGlobalVecOffset(dm,edges[e],ALL_COMPONENTS,&cstart);
    2323:       PetscSectionGetDof(network->DofSection,edges[e],&ncols);

    2325:       if (network->Jv) {
    2326:         Juser = network->Jv[vptr[v-vStart]+2*e+1]; /* Jacobian(v,e) */
    2327:       } else Juser = NULL;
    2328:       MatSetPreallocationblock_private(Juser,nrows,rows_v,ncols,ghost,vd_nz,vo_nz);

    2330:       /* Connected vertices */
    2331:       DMNetworkGetConnectedVertices(dm,edges[e],&cone);
    2332:       vc = (v == cone[0]) ? cone[1]:cone[0];
    2333:       DMNetworkIsGhostVertex(dm,vc,&ghost_vc);

    2335:       PetscSectionGetDof(network->DofSection,vc,&ncols);

    2337:       if (network->Jv) {
    2338:         Juser = network->Jv[vptr[v-vStart]+2*e+2]; /* Jacobian(v,vc) */
    2339:       } else Juser = NULL;
    2340:       if (ghost_vc||ghost) {
    2341:         ghost2 = PETSC_TRUE;
    2342:       } else {
    2343:         ghost2 = PETSC_FALSE;
    2344:       }
    2345:       MatSetPreallocationblock_private(Juser,nrows,rows_v,ncols,ghost2,vd_nz,vo_nz);
    2346:     }

    2348:     /* Set preallocation for vertex self */
    2349:     DMNetworkIsGhostVertex(dm,v,&ghost);
    2350:     if (!ghost) {
    2351:       DMNetworkGetGlobalVecOffset(dm,v,ALL_COMPONENTS,&cstart);
    2352:       if (network->Jv) {
    2353:         Juser = network->Jv[vptr[v-vStart]]; /* Jacobian(v,v) */
    2354:       } else Juser = NULL;
    2355:       MatSetPreallocationblock_private(Juser,nrows,rows_v,nrows,PETSC_FALSE,vd_nz,vo_nz);
    2356:     }
    2357:     if (ghost) {
    2358:       PetscFree(rows_v);
    2359:     }
    2360:   }

    2362:   VecAssemblyBegin(vd_nz);
    2363:   VecAssemblyBegin(vo_nz);

    2365:   PetscMalloc2(localSize,&dnnz,localSize,&onnz);

    2367:   VecAssemblyEnd(vd_nz);
    2368:   VecAssemblyEnd(vo_nz);

    2370:   VecGetArray(vd_nz,&vdnz);
    2371:   VecGetArray(vo_nz,&vonz);
    2372:   for (j=0; j<localSize; j++) {
    2373:     dnnz[j] = (PetscInt)PetscRealPart(vdnz[j]);
    2374:     onnz[j] = (PetscInt)PetscRealPart(vonz[j]);
    2375:   }
    2376:   VecRestoreArray(vd_nz,&vdnz);
    2377:   VecRestoreArray(vo_nz,&vonz);
    2378:   VecDestroy(&vd_nz);
    2379:   VecDestroy(&vo_nz);

    2381:   MatSeqAIJSetPreallocation(*J,0,dnnz);
    2382:   MatMPIAIJSetPreallocation(*J,0,dnnz,0,onnz);
    2383:   MatSetOption(*J,MAT_NEW_NONZERO_ALLOCATION_ERR,PETSC_TRUE);

    2385:   PetscFree2(dnnz,onnz);

    2387:   /* (2) Set matrix entries for edges */
    2388:   /*----------------------------------*/
    2389:   for (e=eStart; e<eEnd; e++) {
    2390:     /* Get row indices */
    2391:     DMNetworkGetGlobalVecOffset(dm,e,ALL_COMPONENTS,&rstart);
    2392:     PetscSectionGetDof(network->DofSection,e,&nrows);
    2393:     if (nrows) {
    2394:       for (j=0; j<nrows; j++) rows[j] = j + rstart;

    2396:       /* Set matrix entries for connected vertices */
    2397:       DMNetworkGetConnectedVertices(dm,e,&cone);
    2398:       for (v=0; v<2; v++) {
    2399:         DMNetworkGetGlobalVecOffset(dm,cone[v],ALL_COMPONENTS,&cstart);
    2400:         PetscSectionGetDof(network->DofSection,cone[v],&ncols);

    2402:         if (network->Je) {
    2403:           Juser = network->Je[3*e+1+v]; /* Jacobian(e,v) */
    2404:         } else Juser = NULL;
    2405:         MatSetblock_private(Juser,nrows,rows,ncols,cstart,J);
    2406:       }

    2408:       /* Set matrix entries for edge self */
    2409:       cstart = rstart;
    2410:       if (network->Je) {
    2411:         Juser = network->Je[3*e]; /* Jacobian(e,e) */
    2412:       } else Juser = NULL;
    2413:       MatSetblock_private(Juser,nrows,rows,nrows,cstart,J);
    2414:     }
    2415:   }

    2417:   /* Set matrix entries for vertices */
    2418:   /*---------------------------------*/
    2419:   for (v=vStart; v<vEnd; v++) {
    2420:     /* Get row indices */
    2421:     DMNetworkGetGlobalVecOffset(dm,v,ALL_COMPONENTS,&rstart);
    2422:     PetscSectionGetDof(network->DofSection,v,&nrows);
    2423:     if (!nrows) continue;

    2425:     DMNetworkIsGhostVertex(dm,v,&ghost);
    2426:     if (ghost) {
    2427:       PetscMalloc1(nrows,&rows_v);
    2428:     } else {
    2429:       rows_v = rows;
    2430:     }
    2431:     for (j=0; j<nrows; j++) rows_v[j] = j + rstart;

    2433:     /* Get supporting edges and connected vertices */
    2434:     DMNetworkGetSupportingEdges(dm,v,&nedges,&edges);

    2436:     for (e=0; e<nedges; e++) {
    2437:       /* Supporting edges */
    2438:       DMNetworkGetGlobalVecOffset(dm,edges[e],ALL_COMPONENTS,&cstart);
    2439:       PetscSectionGetDof(network->DofSection,edges[e],&ncols);

    2441:       if (network->Jv) {
    2442:         Juser = network->Jv[vptr[v-vStart]+2*e+1]; /* Jacobian(v,e) */
    2443:       } else Juser = NULL;
    2444:       MatSetblock_private(Juser,nrows,rows_v,ncols,cstart,J);

    2446:       /* Connected vertices */
    2447:       DMNetworkGetConnectedVertices(dm,edges[e],&cone);
    2448:       vc = (v == cone[0]) ? cone[1]:cone[0];

    2450:       DMNetworkGetGlobalVecOffset(dm,vc,ALL_COMPONENTS,&cstart);
    2451:       PetscSectionGetDof(network->DofSection,vc,&ncols);

    2453:       if (network->Jv) {
    2454:         Juser = network->Jv[vptr[v-vStart]+2*e+2]; /* Jacobian(v,vc) */
    2455:       } else Juser = NULL;
    2456:       MatSetblock_private(Juser,nrows,rows_v,ncols,cstart,J);
    2457:     }

    2459:     /* Set matrix entries for vertex self */
    2460:     if (!ghost) {
    2461:       DMNetworkGetGlobalVecOffset(dm,v,ALL_COMPONENTS,&cstart);
    2462:       if (network->Jv) {
    2463:         Juser = network->Jv[vptr[v-vStart]]; /* Jacobian(v,v) */
    2464:       } else Juser = NULL;
    2465:       MatSetblock_private(Juser,nrows,rows_v,nrows,cstart,J);
    2466:     }
    2467:     if (ghost) {
    2468:       PetscFree(rows_v);
    2469:     }
    2470:   }
    2471:   PetscFree(rows);

    2473:   MatAssemblyBegin(*J,MAT_FINAL_ASSEMBLY);
    2474:   MatAssemblyEnd(*J,MAT_FINAL_ASSEMBLY);

    2476:   MatSetDM(*J,dm);
    2477:   return 0;
    2478: }

    2480: PetscErrorCode DMDestroy_Network(DM dm)
    2481: {
    2482:   DM_Network     *network = (DM_Network*)dm->data;
    2483:   PetscInt       j,np;

    2485:   if (--network->refct > 0) return 0;
    2486:   PetscFree(network->Je);
    2487:   if (network->Jv) {
    2488:     PetscFree(network->Jvptr);
    2489:     PetscFree(network->Jv);
    2490:   }

    2492:   ISLocalToGlobalMappingDestroy(&network->vertex.mapping);
    2493:   PetscSectionDestroy(&network->vertex.DofSection);
    2494:   PetscSectionDestroy(&network->vertex.GlobalDofSection);
    2495:   PetscFree(network->vltog);
    2496:   PetscSFDestroy(&network->vertex.sf);
    2497:   /* edge */
    2498:   ISLocalToGlobalMappingDestroy(&network->edge.mapping);
    2499:   PetscSectionDestroy(&network->edge.DofSection);
    2500:   PetscSectionDestroy(&network->edge.GlobalDofSection);
    2501:   PetscSFDestroy(&network->edge.sf);
    2502:   DMDestroy(&network->plex);
    2503:   PetscSectionDestroy(&network->DataSection);
    2504:   PetscSectionDestroy(&network->DofSection);

    2506:   for (j=0; j<network->Nsvtx; j++) PetscFree(network->svtx[j].sv);
    2507:   PetscFree(network->svtx);
    2508:   PetscFree2(network->subnetedge,network->subnetvtx);

    2510:   PetscTableDestroy(&network->svtable);
    2511:   PetscFree(network->subnet);
    2512:   PetscFree(network->component);
    2513:   PetscFree(network->componentdataarray);

    2515:   if (network->header) {
    2516:     np = network->pEnd - network->pStart;
    2517:     for (j=0; j < np; j++) {
    2518:       PetscFree5(network->header[j].size,network->header[j].key,network->header[j].offset,network->header[j].nvar,network->header[j].offsetvarrel);
    2519:       PetscFree(network->cvalue[j].data);
    2520:     }
    2521:     PetscFree2(network->header,network->cvalue);
    2522:   }
    2523:   PetscFree(network);
    2524:   return 0;
    2525: }

    2527: PetscErrorCode DMView_Network(DM dm,PetscViewer viewer)
    2528: {
    2529:   PetscBool      iascii;
    2530:   PetscMPIInt    rank;

    2534:   MPI_Comm_rank(PetscObjectComm((PetscObject)dm),&rank);
    2535:   PetscObjectTypeCompare((PetscObject)viewer,PETSCVIEWERASCII,&iascii);
    2536:   if (iascii) {
    2537:     const PetscInt *cone,*vtx,*edges;
    2538:     PetscInt       vfrom,vto,i,j,nv,ne,nsv,p,nsubnet;
    2539:     DM_Network     *network = (DM_Network*)dm->data;

    2541:     nsubnet = network->Nsubnet; /* num of subnetworks */
    2542:     if (rank == 0) {
    2543:       PetscPrintf(PETSC_COMM_SELF,"  NSubnets: %" PetscInt_FMT "; NEdges: %" PetscInt_FMT "; NVertices: %" PetscInt_FMT "; NSharedVertices: %" PetscInt_FMT ".\n",nsubnet,network->NEdges,network->NVertices,network->Nsvtx);
    2544:     }

    2546:     DMNetworkGetSharedVertices(dm,&nsv,NULL);
    2547:     PetscViewerASCIIPushSynchronized(viewer);
    2548:     PetscViewerASCIISynchronizedPrintf(viewer, "  [%d] nEdges: %" PetscInt_FMT "; nVertices: %" PetscInt_FMT "; nSharedVertices: %" PetscInt_FMT "\n",rank,network->nEdges,network->nVertices,nsv);

    2550:     for (i=0; i<nsubnet; i++) {
    2551:       DMNetworkGetSubnetwork(dm,i,&nv,&ne,&vtx,&edges);
    2552:       if (ne) {
    2553:         PetscViewerASCIISynchronizedPrintf(viewer, "     Subnet %" PetscInt_FMT ": nEdges %" PetscInt_FMT ", nVertices(include shared vertices) %" PetscInt_FMT "\n",i,ne,nv);
    2554:         for (j=0; j<ne; j++) {
    2555:           p = edges[j];
    2556:           DMNetworkGetConnectedVertices(dm,p,&cone);
    2557:           DMNetworkGetGlobalVertexIndex(dm,cone[0],&vfrom);
    2558:           DMNetworkGetGlobalVertexIndex(dm,cone[1],&vto);
    2559:           DMNetworkGetGlobalEdgeIndex(dm,edges[j],&p);
    2560:           PetscViewerASCIISynchronizedPrintf(viewer, "       edge %" PetscInt_FMT ": %" PetscInt_FMT " ----> %" PetscInt_FMT "\n",p,vfrom,vto);
    2561:         }
    2562:       }
    2563:     }

    2565:     /* Shared vertices */
    2566:     DMNetworkGetSharedVertices(dm,NULL,&vtx);
    2567:     if (nsv) {
    2568:       PetscInt       gidx;
    2569:       PetscBool      ghost;
    2570:       const PetscInt *sv=NULL;

    2572:       PetscViewerASCIISynchronizedPrintf(viewer, "     SharedVertices:\n");
    2573:       for (i=0; i<nsv; i++) {
    2574:         DMNetworkIsGhostVertex(dm,vtx[i],&ghost);
    2575:         if (ghost) continue;

    2577:         DMNetworkSharedVertexGetInfo(dm,vtx[i],&gidx,&nv,&sv);
    2578:         PetscViewerASCIISynchronizedPrintf(viewer, "       svtx %" PetscInt_FMT ": global index %" PetscInt_FMT ", subnet[%" PetscInt_FMT "].%" PetscInt_FMT " ---->\n",i,gidx,sv[0],sv[1]);
    2579:         for (j=1; j<nv; j++) {
    2580:           PetscViewerASCIISynchronizedPrintf(viewer, "                                           ----> subnet[%" PetscInt_FMT "].%" PetscInt_FMT "\n",sv[2*j],sv[2*j+1]);
    2581:         }
    2582:       }
    2583:     }
    2584:     PetscViewerFlush(viewer);
    2585:     PetscViewerASCIIPopSynchronized(viewer);
    2587:   return 0;
    2588: }

    2590: PetscErrorCode DMGlobalToLocalBegin_Network(DM dm, Vec g, InsertMode mode, Vec l)
    2591: {
    2592:   DM_Network     *network = (DM_Network*)dm->data;

    2594:   DMGlobalToLocalBegin(network->plex,g,mode,l);
    2595:   return 0;
    2596: }

    2598: PetscErrorCode DMGlobalToLocalEnd_Network(DM dm, Vec g, InsertMode mode, Vec l)
    2599: {
    2600:   DM_Network     *network = (DM_Network*)dm->data;

    2602:   DMGlobalToLocalEnd(network->plex,g,mode,l);
    2603:   return 0;
    2604: }

    2606: PetscErrorCode DMLocalToGlobalBegin_Network(DM dm, Vec l, InsertMode mode, Vec g)
    2607: {
    2608:   DM_Network     *network = (DM_Network*)dm->data;

    2610:   DMLocalToGlobalBegin(network->plex,l,mode,g);
    2611:   return 0;
    2612: }

    2614: PetscErrorCode DMLocalToGlobalEnd_Network(DM dm, Vec l, InsertMode mode, Vec g)
    2615: {
    2616:   DM_Network     *network = (DM_Network*)dm->data;

    2618:   DMLocalToGlobalEnd(network->plex,l,mode,g);
    2619:   return 0;
    2620: }

    2622: /*@
    2623:   DMNetworkGetVertexLocalToGlobalOrdering - Get vertex global index

    2625:   Not collective

    2627:   Input Parameters:
    2628: + dm - the dm object
    2629: - vloc - local vertex ordering, start from 0

    2631:   Output Parameters:
    2632: .  vg  - global vertex ordering, start from 0

    2634:   Level: advanced

    2636: .seealso: DMNetworkSetVertexLocalToGlobalOrdering()
    2637: @*/
    2638: PetscErrorCode DMNetworkGetVertexLocalToGlobalOrdering(DM dm,PetscInt vloc,PetscInt *vg)
    2639: {
    2640:   DM_Network  *network = (DM_Network*)dm->data;
    2641:   PetscInt    *vltog = network->vltog;

    2644:   *vg = vltog[vloc];
    2645:   return 0;
    2646: }

    2648: /*@
    2649:   DMNetworkSetVertexLocalToGlobalOrdering - Create and setup vertex local to global map

    2651:   Collective

    2653:   Input Parameters:
    2654: . dm - the dm object

    2656:   Level: advanced

    2658: .seealso: DMNetworkGetGlobalVertexIndex()
    2659: @*/
    2660: PetscErrorCode DMNetworkSetVertexLocalToGlobalOrdering(DM dm)
    2661: {
    2662:   DM_Network        *network=(DM_Network*)dm->data;
    2663:   MPI_Comm          comm;
    2664:   PetscMPIInt       rank,size,*displs=NULL,*recvcounts=NULL,remoterank;
    2665:   PetscBool         ghost;
    2666:   PetscInt          *vltog,nroots,nleaves,i,*vrange,k,N,lidx;
    2667:   const PetscSFNode *iremote;
    2668:   PetscSF           vsf;
    2669:   Vec               Vleaves,Vleaves_seq;
    2670:   VecScatter        ctx;
    2671:   PetscScalar       *varr,val;
    2672:   const PetscScalar *varr_read;

    2674:   PetscObjectGetComm((PetscObject)dm,&comm);
    2675:   MPI_Comm_size(comm,&size);
    2676:   MPI_Comm_rank(comm,&rank);

    2678:   if (size == 1) {
    2679:     nroots = network->vEnd - network->vStart;
    2680:     PetscMalloc1(nroots, &vltog);
    2681:     for (i=0; i<nroots; i++) vltog[i] = i;
    2682:     network->vltog = vltog;
    2683:     return 0;
    2684:   }

    2687:   if (network->vltog) {
    2688:     PetscFree(network->vltog);
    2689:   }

    2691:   DMNetworkSetSubMap_private(network->vStart,network->vEnd,&network->vertex.mapping);
    2692:   PetscSFGetSubSF(network->plex->sf, network->vertex.mapping, &network->vertex.sf);
    2693:   vsf = network->vertex.sf;

    2695:   PetscMalloc3(size+1,&vrange,size,&displs,size,&recvcounts);
    2696:   PetscSFGetGraph(vsf,&nroots,&nleaves,NULL,&iremote);

    2698:   for (i=0; i<size; i++) { displs[i] = i; recvcounts[i] = 1;}

    2700:   i         = nroots - nleaves; /* local number of vertices, excluding ghosts */
    2701:   vrange[0] = 0;
    2702:   MPI_Allgatherv(&i,1,MPIU_INT,vrange+1,recvcounts,displs,MPIU_INT,comm);
    2703:   for (i=2; i<size+1; i++) {vrange[i] += vrange[i-1];}

    2705:   PetscMalloc1(nroots, &vltog);
    2706:   network->vltog = vltog;

    2708:   /* Set vltog for non-ghost vertices */
    2709:   k = 0;
    2710:   for (i=0; i<nroots; i++) {
    2711:     DMNetworkIsGhostVertex(dm,i+network->vStart,&ghost);
    2712:     if (ghost) continue;
    2713:     vltog[i] = vrange[rank] + k++;
    2714:   }
    2715:   PetscFree3(vrange,displs,recvcounts);

    2717:   /* Set vltog for ghost vertices */
    2718:   /* (a) create parallel Vleaves and sequential Vleaves_seq to convert local iremote[*].index to global index */
    2719:   VecCreate(comm,&Vleaves);
    2720:   VecSetSizes(Vleaves,2*nleaves,PETSC_DETERMINE);
    2721:   VecSetFromOptions(Vleaves);
    2722:   VecGetArray(Vleaves,&varr);
    2723:   for (i=0; i<nleaves; i++) {
    2724:     varr[2*i]   = (PetscScalar)(iremote[i].rank);  /* rank of remote process */
    2725:     varr[2*i+1] = (PetscScalar)(iremote[i].index); /* local index in remote process */
    2726:   }
    2727:   VecRestoreArray(Vleaves,&varr);

    2729:   /* (b) scatter local info to remote processes via VecScatter() */
    2730:   VecScatterCreateToAll(Vleaves,&ctx,&Vleaves_seq);
    2731:   VecScatterBegin(ctx,Vleaves,Vleaves_seq,INSERT_VALUES,SCATTER_FORWARD);
    2732:   VecScatterEnd(ctx,Vleaves,Vleaves_seq,INSERT_VALUES,SCATTER_FORWARD);

    2734:   /* (c) convert local indices to global indices in parallel vector Vleaves */
    2735:   VecGetSize(Vleaves_seq,&N);
    2736:   VecGetArrayRead(Vleaves_seq,&varr_read);
    2737:   for (i=0; i<N; i+=2) {
    2738:     remoterank = (PetscMPIInt)PetscRealPart(varr_read[i]);
    2739:     if (remoterank == rank) {
    2740:       k = i+1; /* row number */
    2741:       lidx = (PetscInt)PetscRealPart(varr_read[i+1]);
    2742:       val  = (PetscScalar)vltog[lidx]; /* global index for non-ghost vertex computed above */
    2743:       VecSetValues(Vleaves,1,&k,&val,INSERT_VALUES);
    2744:     }
    2745:   }
    2746:   VecRestoreArrayRead(Vleaves_seq,&varr_read);
    2747:   VecAssemblyBegin(Vleaves);
    2748:   VecAssemblyEnd(Vleaves);

    2750:   /* (d) Set vltog for ghost vertices by copying local values of Vleaves */
    2751:   VecGetArrayRead(Vleaves,&varr_read);
    2752:   k = 0;
    2753:   for (i=0; i<nroots; i++) {
    2754:     DMNetworkIsGhostVertex(dm,i+network->vStart,&ghost);
    2755:     if (!ghost) continue;
    2756:     vltog[i] = (PetscInt)PetscRealPart(varr_read[2*k+1]); k++;
    2757:   }
    2758:   VecRestoreArrayRead(Vleaves,&varr_read);

    2760:   VecDestroy(&Vleaves);
    2761:   VecDestroy(&Vleaves_seq);
    2762:   VecScatterDestroy(&ctx);
    2763:   return 0;
    2764: }

    2766: static inline PetscErrorCode DMISAddSize_private(DM_Network *network,PetscInt p,PetscInt numkeys,PetscInt keys[],PetscInt blocksize[],PetscInt nselectedvar[],PetscInt *nidx)
    2767: {
    2768:   PetscInt                 i,j,ncomps,nvar,key,offset=0;
    2769:   DMNetworkComponentHeader header;

    2771:   PetscSectionGetOffset(network->DataSection,p,&offset);
    2772:   ncomps = ((DMNetworkComponentHeader)(network->componentdataarray+offset))->ndata;
    2773:   header = (DMNetworkComponentHeader)(network->componentdataarray+offset);

    2775:   for (i=0; i<ncomps; i++) {
    2776:     key  = header->key[i];
    2777:     nvar = header->nvar[i];
    2778:     for (j=0; j<numkeys; j++) {
    2779:       if (key == keys[j]) {
    2780:         if (!blocksize || blocksize[j] == -1) {
    2781:           *nidx += nvar;
    2782:         } else {
    2783:           *nidx += nselectedvar[j]*nvar/blocksize[j];
    2784:         }
    2785:       }
    2786:     }
    2787:   }
    2788:   return 0;
    2789: }

    2791: static inline PetscErrorCode DMISComputeIdx_private(DM dm,PetscInt p,PetscInt numkeys,PetscInt keys[],PetscInt blocksize[],PetscInt nselectedvar[],PetscInt *selectedvar[],PetscInt *ii,PetscInt *idx)
    2792: {
    2793:   PetscInt                 i,j,ncomps,nvar,key,offsetg,k,k1,offset=0;
    2794:   DM_Network               *network = (DM_Network*)dm->data;
    2795:   DMNetworkComponentHeader header;

    2797:   PetscSectionGetOffset(network->DataSection,p,&offset);
    2798:   ncomps = ((DMNetworkComponentHeader)(network->componentdataarray+offset))->ndata;
    2799:   header = (DMNetworkComponentHeader)(network->componentdataarray+offset);

    2801:   for (i=0; i<ncomps; i++) {
    2802:     key  = header->key[i];
    2803:     nvar = header->nvar[i];
    2804:     for (j=0; j<numkeys; j++) {
    2805:       if (key != keys[j]) continue;

    2807:       DMNetworkGetGlobalVecOffset(dm,p,i,&offsetg);
    2808:       if (!blocksize || blocksize[j] == -1) {
    2809:         for (k=0; k<nvar; k++) idx[(*ii)++] = offsetg + k;
    2810:       } else {
    2811:         for (k=0; k<nvar; k+=blocksize[j]) {
    2812:           for (k1=0; k1<nselectedvar[j]; k1++) idx[(*ii)++] = offsetg + k + selectedvar[j][k1];
    2813:         }
    2814:       }
    2815:     }
    2816:   }
    2817:   return 0;
    2818: }

    2820: /*@
    2821:   DMNetworkCreateIS - Create an index set object from the global vector of the network

    2823:   Collective

    2825:   Input Parameters:
    2826: + dm - DMNetwork object
    2827: . numkeys - number of keys
    2828: . keys - array of keys that define the components of the variables you wish to extract
    2829: . blocksize - block size of the variables associated to the component
    2830: . nselectedvar - number of variables in each block to select
    2831: - selectedvar - the offset into the block of each variable in each block to select

    2833:   Output Parameters:
    2834: . is - the index set

    2836:   Level: Advanced

    2838:   Notes:
    2839:     Use blocksize[i] of -1 to indicate select all the variables of the i-th component, for which nselectvar[i] and selectedvar[i] are ignored. Use NULL, NULL, NULL to indicate for all selected components one wishes to obtain all the values of that component. For example, DMNetworkCreateIS(dm,1,&key,NULL,NULL,NULL,&is) will return an is that extracts all the variables for the 0-th component.

    2841: .seealso: DMNetworkCreate(), ISCreateGeneral(), DMNetworkCreateLocalIS()
    2842: @*/
    2843: PetscErrorCode DMNetworkCreateIS(DM dm,PetscInt numkeys,PetscInt keys[],PetscInt blocksize[],PetscInt nselectedvar[],PetscInt *selectedvar[],IS *is)
    2844: {
    2845:   MPI_Comm       comm;
    2846:   DM_Network     *network = (DM_Network*)dm->data;
    2847:   PetscInt       i,p,estart,eend,vstart,vend,nidx,*idx;
    2848:   PetscBool      ghost;

    2850:   PetscObjectGetComm((PetscObject)dm,&comm);

    2852:   /* Check input parameters */
    2853:   for (i=0; i<numkeys; i++) {
    2854:     if (!blocksize || blocksize[i] == -1) continue;
    2856:   }

    2858:   DMNetworkGetEdgeRange(dm,&estart,&eend);
    2859:   DMNetworkGetVertexRange(dm,&vstart,&vend);

    2861:   /* Get local number of idx */
    2862:   nidx = 0;
    2863:   for (p=estart; p<eend; p++) {
    2864:     DMISAddSize_private(network,p,numkeys,keys,blocksize,nselectedvar,&nidx);
    2865:   }
    2866:   for (p=vstart; p<vend; p++) {
    2867:     DMNetworkIsGhostVertex(dm,p,&ghost);
    2868:     if (ghost) continue;
    2869:     DMISAddSize_private(network,p,numkeys,keys,blocksize,nselectedvar,&nidx);
    2870:   }

    2872:   /* Compute idx */
    2873:   PetscMalloc1(nidx,&idx);
    2874:   i = 0;
    2875:   for (p=estart; p<eend; p++) {
    2876:     DMISComputeIdx_private(dm,p,numkeys,keys,blocksize,nselectedvar,selectedvar,&i,idx);
    2877:   }
    2878:   for (p=vstart; p<vend; p++) {
    2879:     DMNetworkIsGhostVertex(dm,p,&ghost);
    2880:     if (ghost) continue;
    2881:     DMISComputeIdx_private(dm,p,numkeys,keys,blocksize,nselectedvar,selectedvar,&i,idx);
    2882:   }

    2884:   /* Create is */
    2885:   ISCreateGeneral(comm,nidx,idx,PETSC_COPY_VALUES,is);
    2886:   PetscFree(idx);
    2887:   return 0;
    2888: }

    2890: static inline PetscErrorCode DMISComputeLocalIdx_private(DM dm,PetscInt p,PetscInt numkeys,PetscInt keys[],PetscInt blocksize[],PetscInt nselectedvar[],PetscInt *selectedvar[],PetscInt *ii,PetscInt *idx)
    2891: {
    2892:   PetscInt                 i,j,ncomps,nvar,key,offsetl,k,k1,offset=0;
    2893:   DM_Network               *network = (DM_Network*)dm->data;
    2894:   DMNetworkComponentHeader header;

    2896:   PetscSectionGetOffset(network->DataSection,p,&offset);
    2897:   ncomps = ((DMNetworkComponentHeader)(network->componentdataarray+offset))->ndata;
    2898:   header = (DMNetworkComponentHeader)(network->componentdataarray+offset);

    2900:   for (i=0; i<ncomps; i++) {
    2901:     key  = header->key[i];
    2902:     nvar = header->nvar[i];
    2903:     for (j=0; j<numkeys; j++) {
    2904:       if (key != keys[j]) continue;

    2906:       DMNetworkGetLocalVecOffset(dm,p,i,&offsetl);
    2907:       if (!blocksize || blocksize[j] == -1) {
    2908:         for (k=0; k<nvar; k++) idx[(*ii)++] = offsetl + k;
    2909:       } else {
    2910:         for (k=0; k<nvar; k+=blocksize[j]) {
    2911:           for (k1=0; k1<nselectedvar[j]; k1++) idx[(*ii)++] = offsetl + k + selectedvar[j][k1];
    2912:         }
    2913:       }
    2914:     }
    2915:   }
    2916:   return 0;
    2917: }

    2919: /*@
    2920:   DMNetworkCreateLocalIS - Create an index set object from the local vector of the network

    2922:   Not collective

    2924:   Input Parameters:
    2925: + dm - DMNetwork object
    2926: . numkeys - number of keys
    2927: . keys - array of keys that define the components of the variables you wish to extract
    2928: . blocksize - block size of the variables associated to the component
    2929: . nselectedvar - number of variables in each block to select
    2930: - selectedvar - the offset into the block of each variable in each block to select

    2932:   Output Parameters:
    2933: . is - the index set

    2935:   Level: Advanced

    2937:   Notes:
    2938:     Use blocksize[i] of -1 to indicate select all the variables of the i-th component, for which nselectvar[i] and selectedvar[i] are ignored. Use NULL, NULL, NULL to indicate for all selected components one wishes to obtain all the values of that component. For example, DMNetworkCreateLocalIS(dm,1,&key,NULL,NULL,NULL,&is) will return an is that extracts all the variables for the 0-th component.

    2940: .seealso: DMNetworkCreate(), DMNetworkCreateIS, ISCreateGeneral()
    2941: @*/
    2942: PetscErrorCode DMNetworkCreateLocalIS(DM dm,PetscInt numkeys,PetscInt keys[],PetscInt blocksize[],PetscInt nselectedvar[],PetscInt *selectedvar[],IS *is)
    2943: {
    2944:   DM_Network     *network = (DM_Network*)dm->data;
    2945:   PetscInt       i,p,pstart,pend,nidx,*idx;

    2947:   /* Check input parameters */
    2948:   for (i=0; i<numkeys; i++) {
    2949:     if (!blocksize || blocksize[i] == -1) continue;
    2951:   }

    2953:   pstart = network->pStart;
    2954:   pend   = network->pEnd;

    2956:   /* Get local number of idx */
    2957:   nidx = 0;
    2958:   for (p=pstart; p<pend; p++) {
    2959:     DMISAddSize_private(network,p,numkeys,keys,blocksize,nselectedvar,&nidx);
    2960:   }

    2962:   /* Compute local idx */
    2963:   PetscMalloc1(nidx,&idx);
    2964:   i = 0;
    2965:   for (p=pstart; p<pend; p++) {
    2966:     DMISComputeLocalIdx_private(dm,p,numkeys,keys,blocksize,nselectedvar,selectedvar,&i,idx);
    2967:   }

    2969:   /* Create is */
    2970:   ISCreateGeneral(PETSC_COMM_SELF,nidx,idx,PETSC_COPY_VALUES,is);
    2971:   PetscFree(idx);
    2972:   return 0;
    2973: }