Date: 2025-03-29T05:58:20+00:00

Actual source code: network.c

      1: #include <petsc/private/dmnetworkimpl.h>

      3: PetscLogEvent DMNetwork_LayoutSetUp;
      4: PetscLogEvent DMNetwork_SetUpNetwork;
      5: PetscLogEvent DMNetwork_Distribute;

      7: /*
      8:   Creates the component header and value objects for a network point
      9: */
     10: static PetscErrorCode SetUpNetworkHeaderComponentValue(DM dm, DMNetworkComponentHeader header, DMNetworkComponentValue cvalue)
     11: {
     12:   PetscFunctionBegin;
     13:   /* Allocate arrays for component information */
     14:   PetscCall(PetscCalloc5(header->maxcomps, &header->size, header->maxcomps, &header->key, header->maxcomps, &header->offset, header->maxcomps, &header->nvar, header->maxcomps, &header->offsetvarrel));
     15:   PetscCall(PetscCalloc1(header->maxcomps, &cvalue->data));

     17:   /* The size of the header is the size of struct _p_DMNetworkComponentHeader. Since the struct contains PetscInt pointers we cannot use sizeof(struct). So, we need to explicitly calculate the size.
     18:    If the data header struct changes then this header size calculation needs to be updated. */
     19:   header->hsize = sizeof(struct _p_DMNetworkComponentHeader) + 5 * header->maxcomps * sizeof(PetscInt);
     20:   header->hsize /= sizeof(DMNetworkComponentGenericDataType);
     21: #if defined(__NEC__)
     22:   /* NEC/LG: quick hack to keep data aligned on 8 bytes. */
     23:   header->hsize = (header->hsize + (8 - 1)) & ~(8 - 1);
     24: #endif
     25:   PetscFunctionReturn(PETSC_SUCCESS);
     26: }

     28: PetscErrorCode DMNetworkInitializeHeaderComponentData(DM dm)
     29: {
     30:   DM_Network *network = (DM_Network *)dm->data;
     31:   PetscInt    np, p, defaultnumcomp = DMNETWORK_MAX_COMP_AT_POINT_DEFAULT;

     33:   PetscFunctionBegin;
     34:   np = network->cloneshared->pEnd - network->cloneshared->pStart;
     35:   if (network->header)
     36:     for (p = 0; p < np; p++) {
     37:       network->header[p].maxcomps = defaultnumcomp;
     38:       PetscCall(SetUpNetworkHeaderComponentValue(dm, &network->header[p], &network->cvalue[p]));
     39:     }
     40:   PetscFunctionReturn(PETSC_SUCCESS);
     41: }

     43: /*@
     44:   DMNetworkGetPlex - Gets the `DMPLEX` associated with this `DMNETWORK`

     46:   Not Collective

     48:   Input Parameter:
     49: . dm - the `DMNETWORK` object

     51:   Output Parameter:
     52: . plexdm - the `DMPLEX` object

     54:   Level: advanced

     56: .seealso: `DM`, `DMNETWORK`, `DMPLEX`, `DMNetworkCreate()`
     57: @*/
     58: PetscErrorCode DMNetworkGetPlex(DM dm, DM *plexdm)
     59: {
     60:   DM_Network *network = (DM_Network *)dm->data;

     62:   PetscFunctionBegin;
     63:   *plexdm = network->plex;
     64:   PetscFunctionReturn(PETSC_SUCCESS);
     65: }

     67: /*@
     68:   DMNetworkGetNumSubNetworks - Gets the number of subnetworks

     70:   Not Collective

     72:   Input Parameter:
     73: . dm - the `DMNETWORK` object

     75:   Output Parameters:
     76: + nsubnet - local number of subnetworks, pass `NULL` if not needed
     77: - Nsubnet - global number of subnetworks, pass `NULL` if not needed

     79:   Level: beginner

     81: .seealso: `DM`, `DMNETWORK`, `DMNetworkCreate()`, `DMNetworkSetNumSubNetworks()`
     82: @*/
     83: PetscErrorCode DMNetworkGetNumSubNetworks(DM dm, PeOp PetscInt *nsubnet, PeOp PetscInt *Nsubnet)
     84: {
     85:   DM_Network *network = (DM_Network *)dm->data;

     87:   PetscFunctionBegin;
     88:   if (nsubnet) *nsubnet = network->cloneshared->nsubnet;
     89:   if (Nsubnet) *Nsubnet = network->cloneshared->Nsubnet;
     90:   PetscFunctionReturn(PETSC_SUCCESS);
     91: }

     93: /*@
     94:   DMNetworkSetNumSubNetworks - Sets the number of subnetworks

     96:   Collective

     98:   Input Parameters:
     99: + dm      - the `DMNETWORK` object
    100: . nsubnet - local number of subnetworks
    101: - Nsubnet - global number of subnetworks

    103:   Level: beginner

    105: .seealso: `DM`, `DMNETWORK`, `DMNetworkCreate()`, `DMNetworkGetNumSubNetworks()`
    106: @*/
    107: PetscErrorCode DMNetworkSetNumSubNetworks(DM dm, PetscInt nsubnet, PetscInt Nsubnet)
    108: {
    109:   DM_Network *network = (DM_Network *)dm->data;

    111:   PetscFunctionBegin;
    112:   PetscCheck(network->cloneshared->Nsubnet == 0, PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_INCOMP, "Network sizes already set, cannot resize the network");

    118:   if (Nsubnet == PETSC_DECIDE) {
    119:     PetscCheck(nsubnet >= 0, PETSC_COMM_SELF, PETSC_ERR_ARG_INCOMP, "Number of local subnetworks %" PetscInt_FMT " cannot be less than 0", nsubnet);
    120:     PetscCallMPI(MPIU_Allreduce(&nsubnet, &Nsubnet, 1, MPIU_INT, MPI_SUM, PetscObjectComm((PetscObject)dm)));
    121:   }
    122:   PetscCheck(Nsubnet >= 1, PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_INCOMP, "Number of global subnetworks %" PetscInt_FMT " cannot be less than 1", Nsubnet);

    124:   network->cloneshared->Nsubnet = Nsubnet;
    125:   network->cloneshared->nsubnet = 0; /* initial value; will be determined by DMNetworkAddSubnetwork() */
    126:   PetscCall(PetscCalloc1(Nsubnet, &network->cloneshared->subnet));

    128:   /* num of shared vertices */
    129:   network->cloneshared->nsvtx = 0;
    130:   network->cloneshared->Nsvtx = 0;
    131:   PetscFunctionReturn(PETSC_SUCCESS);
    132: }

    134: /*@
    135:   DMNetworkAddSubnetwork - Add a subnetwork

    137:   Collective

    139:   Input Parameters:
    140: + dm       - the `DMNETWORK`  object
    141: . name     - name of the subnetwork
    142: . ne       - number of local edges of this subnetwork
    143: - edgelist - list of edges for this subnetwork, this is a one dimensional array with pairs of entries being the two vertices (in global numbering
    144:               of the vertices) of each edge,
    145: $            [first vertex of first edge, second vertex of first edge, first vertex of second edge, second vertex of second edge, etc]

    147:   Output Parameter:
    148: . netnum - global index of the subnetwork

    150:   Level: beginner

    152:   Notes:
    153:   There is no copy involved in this operation, only the pointer is referenced. The `edgelist` should
    154:   not be destroyed before the call to `DMNetworkLayoutSetUp()`

    156:   A network can comprise of a single subnetwork OR multiple subnetworks. For a single subnetwork, the subnetwork can be read either in serial or parallel.
    157:   For a multiple subnetworks,
    158:   each subnetwork topology needs to be set on a unique MPI process and the communicator size needs to be at least equal to the number of subnetworks.

    160:   Example usage:
    161:   Consider the following networks\:
    162:   1) A single subnetwork\:
    163: .vb
    164:  network 0:
    165:  rank[0]:
    166:    v0 --> v2; v1 --> v2
    167:  rank[1]:
    168:    v3 --> v5; v4 --> v5
    169: .ve

    171:   The resulting input network 0\:
    172: .vb
    173:   rank[0]:
    174:   ne = 2
    175:   edgelist = [0 2 | 1 2]

    177:   rank[1]:
    178:   ne = 2
    179:   edgelist = [3 5 | 4 5]
    180: .ve
    181:   2) Two subnetworks\:
    182: .vb
    183:  subnetwork 0:
    184:  rank[0]:
    185:    v0 --> v2; v2 --> v1; v1 --> v3;
    186:  subnetwork 1:
    187:  rank[1]:
    188:    v0 --> v3; v3 --> v2; v2 --> v1;
    189: .ve

    191:   The resulting input subnetwork 0\:
    192: .vb
    193:   rank[0]:
    194:   ne = 3
    195:   edgelist = [0 2 | 2 1 | 1 3]

    197:   rank[1]:
    198:   ne = 0
    199:   edgelist = NULL
    200: .ve
    201:   subnetwork 1\:
    202: .vb
    203:   rank[0]:
    204:   ne = 0
    205:   edgelist = NULL

    207:   rank[1]:
    208:   edgelist = [0 3 | 3 2 | 2 1]
    209: .ve

    211: .seealso: `DM`, `DMNETWORK`, `DMNetworkCreate()`, `DMNetworkSetNumSubnetworks()`
    212: @*/
    213: PetscErrorCode DMNetworkAddSubnetwork(DM dm, const char *name, PetscInt ne, PetscInt edgelist[], PetscInt *netnum)
    214: {
    215:   DM_Network *network = (DM_Network *)dm->data;
    216:   PetscInt    i, Nedge, j, Nvtx, nvtx, nvtx_min = -1, nvtx_max = 0;
    217:   PetscBT     table;

    219:   PetscFunctionBegin;
    220:   for (i = 0; i < ne; i++) PetscCheck(edgelist[2 * i] != edgelist[2 * i + 1], PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Edge %" PetscInt_FMT " has the same vertex %" PetscInt_FMT " at each endpoint", i, edgelist[2 * i]);

    222:   i = 0;
    223:   if (ne) nvtx_min = nvtx_max = edgelist[0];
    224:   for (j = 0; j < ne; j++) {
    225:     nvtx_min = PetscMin(nvtx_min, edgelist[i]);
    226:     nvtx_max = PetscMax(nvtx_max, edgelist[i]);
    227:     i++;
    228:     nvtx_min = PetscMin(nvtx_min, edgelist[i]);
    229:     nvtx_max = PetscMax(nvtx_max, edgelist[i]);
    230:     i++;
    231:   }
    232:   Nvtx = nvtx_max - nvtx_min + 1; /* approximated total local nvtx for this subnet */

    234:   /* Get exact local nvtx for this subnet: counting local values between nvtx_min and nvtx_max */
    235:   PetscCall(PetscBTCreate(Nvtx, &table));
    236:   PetscCall(PetscBTMemzero(Nvtx, table));
    237:   i = 0;
    238:   for (j = 0; j < ne; j++) {
    239:     PetscCall(PetscBTSet(table, edgelist[i++] - nvtx_min));
    240:     PetscCall(PetscBTSet(table, edgelist[i++] - nvtx_min));
    241:   }
    242:   nvtx = 0;
    243:   for (j = 0; j < Nvtx; j++) {
    244:     if (PetscBTLookup(table, j)) nvtx++;
    245:   }

    247:   /* Get global total Nvtx = max(edgelist[])+1 for this subnet */
    248:   PetscCallMPI(MPIU_Allreduce(&nvtx_max, &Nvtx, 1, MPIU_INT, MPI_MAX, PetscObjectComm((PetscObject)dm)));
    249:   Nvtx++;
    250:   PetscCall(PetscBTDestroy(&table));

    252:   /* Get global total Nedge for this subnet */
    253:   PetscCallMPI(MPIU_Allreduce(&ne, &Nedge, 1, MPIU_INT, MPI_SUM, PetscObjectComm((PetscObject)dm)));

    255:   i = network->cloneshared->nsubnet;
    256:   if (name) PetscCall(PetscStrncpy(network->cloneshared->subnet[i].name, name, sizeof(network->cloneshared->subnet[i].name)));
    257:   network->cloneshared->subnet[i].nvtx     = nvtx; /* include ghost vertices */
    258:   network->cloneshared->subnet[i].nedge    = ne;
    259:   network->cloneshared->subnet[i].edgelist = edgelist;
    260:   network->cloneshared->subnet[i].Nvtx     = Nvtx;
    261:   network->cloneshared->subnet[i].Nedge    = Nedge;

    263:   /* ----------------------------------------------------------
    264:    p=v or e;
    265:    subnet[0].pStart   = 0
    266:    subnet[i+1].pStart = subnet[i].pEnd = subnet[i].pStart + (nE[i] or NV[i])
    267:    ----------------------------------------------------------------------- */
    268:   /* GLOBAL subnet[].vStart and vEnd, used by DMNetworkLayoutSetUp() */
    269:   network->cloneshared->subnet[i].vStart = network->cloneshared->NVertices;
    270:   network->cloneshared->subnet[i].vEnd   = network->cloneshared->subnet[i].vStart + network->cloneshared->subnet[i].Nvtx; /* global vEnd of subnet[i] */

    272:   network->cloneshared->nVertices += nvtx; /* include ghost vertices */
    273:   network->cloneshared->NVertices += network->cloneshared->subnet[i].Nvtx;

    275:   /* LOCAL subnet[].eStart and eEnd, used by DMNetworkLayoutSetUp() */
    276:   network->cloneshared->subnet[i].eStart = network->cloneshared->nEdges;
    277:   network->cloneshared->subnet[i].eEnd   = network->cloneshared->subnet[i].eStart + ne;
    278:   network->cloneshared->nEdges += ne;
    279:   network->cloneshared->NEdges += network->cloneshared->subnet[i].Nedge;

    281:   PetscCall(PetscStrncpy(network->cloneshared->subnet[i].name, name, sizeof(network->cloneshared->subnet[i].name)));
    282:   if (netnum) *netnum = network->cloneshared->nsubnet;
    283:   network->cloneshared->nsubnet++;
    284:   PetscFunctionReturn(PETSC_SUCCESS);
    285: }

    287: /*@C
    288:   DMNetworkSharedVertexGetInfo - Get info of a shared vertex struct, see petsc/private/dmnetworkimpl.h

    290:   Not Collective

    292:   Input Parameters:
    293: + dm - the `DM` object
    294: - v  - vertex point

    296:   Output Parameters:
    297: + gidx - global number of this shared vertex in the internal dmplex, pass `NULL` if not needed
    298: . n    - number of subnetworks that share this vertex, pass `NULL` if not needed
    299: - sv   - array of size `n`: sv[2*i,2*i+1]=(net[i], idx[i]), i=0,...,n-1, pass `NULL` if not needed

    301:   Level: intermediate

    303: .seealso: `DM`, `DMNETWORK`, `DMNetworkGetSharedVertices()`
    304: @*/
    305: PetscErrorCode DMNetworkSharedVertexGetInfo(DM dm, PetscInt v, PeOp PetscInt *gidx, PeOp PetscInt *n, const PeOp PetscInt *sv[])
    306: {
    307:   DM_Network *network = (DM_Network *)dm->data;
    308:   SVtx       *svtx    = network->cloneshared->svtx;
    309:   PetscInt    i, gidx_tmp;

    311:   PetscFunctionBegin;
    312:   PetscCall(DMNetworkGetGlobalVertexIndex(dm, v, &gidx_tmp));
    313:   PetscCall(PetscHMapIGetWithDefault(network->cloneshared->svtable, gidx_tmp + 1, 0, &i));
    314:   PetscCheck(i > 0, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "input vertex is not a shared vertex");

    316:   i--;
    317:   if (gidx) *gidx = gidx_tmp;
    318:   if (n) *n = svtx[i].n;
    319:   if (sv) *sv = svtx[i].sv;
    320:   PetscFunctionReturn(PETSC_SUCCESS);
    321: }

    323: /*
    324:   VtxGetInfo - Get info of an input vertex=(net,idx)

    326:   Input Parameters:
    327: + Nsvtx - global num of shared vertices
    328: . svtx - array of shared vertices (global)
    329: - (net,idx) - subnet number and local index for a vertex

    331:   Output Parameters:
    332: + gidx - global index of (net,idx)
    333: . svtype - see petsc/private/dmnetworkimpl.h
    334: - svtx_idx - ordering in the svtx array
    335: */
    336: static inline PetscErrorCode VtxGetInfo(PetscInt Nsvtx, SVtx *svtx, PetscInt net, PetscInt idx, PetscInt *gidx, SVtxType *svtype, PetscInt *svtx_idx)
    337: {
    338:   PetscInt i, j, *svto, g_idx;
    339:   SVtxType vtype;

    341:   PetscFunctionBegin;
    342:   if (!Nsvtx) PetscFunctionReturn(PETSC_SUCCESS);

    344:   g_idx = -1;
    345:   vtype = SVNONE;

    347:   for (i = 0; i < Nsvtx; i++) {
    348:     if (net == svtx[i].sv[0] && idx == svtx[i].sv[1]) {
    349:       g_idx = svtx[i].gidx;
    350:       vtype = SVFROM;
    351:     } else { /* loop over svtx[i].n */
    352:       for (j = 1; j < svtx[i].n; j++) {
    353:         svto = svtx[i].sv + 2 * j;
    354:         if (net == svto[0] && idx == svto[1]) {
    355:           /* input vertex net.idx is a shared to_vertex, output its global index and its svtype */
    356:           g_idx = svtx[i].gidx; /* output gidx for to_vertex */
    357:           vtype = SVTO;
    358:         }
    359:       }
    360:     }
    361:     if (vtype != SVNONE) break;
    362:   }
    363:   if (gidx) *gidx = g_idx;
    364:   if (svtype) *svtype = vtype;
    365:   if (svtx_idx) *svtx_idx = i;
    366:   PetscFunctionReturn(PETSC_SUCCESS);
    367: }

    369: /*
    370:   TableAddSVtx - Add a new shared vertice from sedgelist[k] to a ctable svta

    372:   Input:  network, sedgelist, k, svta
    373:   Output: svta, tdata, ta2sv
    374: */
    375: static inline PetscErrorCode TableAddSVtx(DM_Network *network, PetscInt *sedgelist, PetscInt k, PetscHMapI svta, PetscInt *tdata, PetscInt *ta2sv)
    376: {
    377:   PetscInt net, idx, gidx;

    379:   PetscFunctionBegin;
    380:   net  = sedgelist[k];
    381:   idx  = sedgelist[k + 1];
    382:   gidx = network->cloneshared->subnet[net].vStart + idx;
    383:   PetscCall(PetscHMapISet(svta, gidx + 1, *tdata + 1));

    385:   ta2sv[*tdata] = k; /* maps tdata to index of sedgelist */
    386:   (*tdata)++;
    387:   PetscFunctionReturn(PETSC_SUCCESS);
    388: }

    390: /*
    391:   SharedVtxCreate - Create an array of global shared vertices. See SVtx and SVtxType in dmnetworkimpl.h

    393:   Input:  dm, Nsedgelist, sedgelist

    395:   Note: Output svtx is organized as
    396:         sv(net[0],idx[0]) --> sv(net[1],idx[1])
    397:                           --> sv(net[1],idx[1])
    398:                           ...
    399:                           --> sv(net[n-1],idx[n-1])
    400:         and net[0] < net[1] < ... < net[n-1]
    401:         where sv[0] has SVFROM type, sv[i], i>0, has SVTO type.
    402:  */
    403: static PetscErrorCode SharedVtxCreate(DM dm, PetscInt Nsedgelist, PetscInt *sedgelist)
    404: {
    405:   SVtx         *svtx = NULL;
    406:   PetscInt     *sv, k, j, nsv, *tdata, **ta2sv;
    407:   PetscHMapI   *svtas;
    408:   PetscInt      gidx, net, idx, i, nta, ita, idx_from, idx_to, n, *net_tmp, *idx_tmp, *gidx_tmp;
    409:   DM_Network   *network = (DM_Network *)dm->data;
    410:   PetscHashIter ppos;

    412:   PetscFunctionBegin;
    413:   /* (1) Crete an array of ctables svtas to map (net,idx) -> gidx; a svtas[] for a shared/merged vertex */
    414:   PetscCall(PetscCalloc3(Nsedgelist, &svtas, Nsedgelist, &tdata, 2 * Nsedgelist, &ta2sv));

    416:   k   = 0; /* sedgelist vertex counter j = 4*k */
    417:   nta = 0; /* num of svta tables created = num of groups of shared vertices */

    419:   /* for j=0 */
    420:   PetscCall(PetscHMapICreateWithSize(2 * Nsedgelist, svtas + nta));
    421:   PetscCall(PetscMalloc1(2 * Nsedgelist, &ta2sv[nta]));

    423:   PetscCall(TableAddSVtx(network, sedgelist, k, svtas[nta], &tdata[nta], ta2sv[nta]));
    424:   PetscCall(TableAddSVtx(network, sedgelist, k + 2, svtas[nta], &tdata[nta], ta2sv[nta]));
    425:   nta++;
    426:   k += 4;

    428:   for (j = 1; j < Nsedgelist; j++) { /* j: sedgelist counter */
    429:     for (ita = 0; ita < nta; ita++) {
    430:       /* vfrom */
    431:       net  = sedgelist[k];
    432:       idx  = sedgelist[k + 1];
    433:       gidx = network->cloneshared->subnet[net].vStart + idx; /* global index of the vertex net.idx before merging shared vertices */
    434:       PetscCall(PetscHMapIGetWithDefault(svtas[ita], gidx + 1, 0, &idx_from));

    436:       /* vto */
    437:       net  = sedgelist[k + 2];
    438:       idx  = sedgelist[k + 3];
    439:       gidx = network->cloneshared->subnet[net].vStart + idx;
    440:       PetscCall(PetscHMapIGetWithDefault(svtas[ita], gidx + 1, 0, &idx_to));

    442:       if (idx_from || idx_to) { /* vfrom or vto is on table svtas[ita] */
    443:         idx_from--;
    444:         idx_to--;
    445:         if (idx_from < 0) { /* vto is on svtas[ita] */
    446:           PetscCall(TableAddSVtx(network, sedgelist, k, svtas[ita], &tdata[ita], ta2sv[ita]));
    447:           break;
    448:         } else if (idx_to < 0) {
    449:           PetscCall(TableAddSVtx(network, sedgelist, k + 2, svtas[ita], &tdata[ita], ta2sv[ita]));
    450:           break;
    451:         }
    452:       }
    453:     }

    455:     if (ita == nta) {
    456:       PetscCall(PetscHMapICreateWithSize(2 * Nsedgelist, svtas + nta));
    457:       PetscCall(PetscMalloc1(2 * Nsedgelist, &ta2sv[nta]));

    459:       PetscCall(TableAddSVtx(network, sedgelist, k, svtas[nta], &tdata[nta], ta2sv[nta]));
    460:       PetscCall(TableAddSVtx(network, sedgelist, k + 2, svtas[nta], &tdata[nta], ta2sv[nta]));
    461:       nta++;
    462:     }
    463:     k += 4;
    464:   }

    466:   /* (2) Create svtable for query shared vertices using gidx */
    467:   PetscCall(PetscHMapICreateWithSize(nta, &network->cloneshared->svtable));

    469:   /* (3) Construct svtx from svtas
    470:    svtx: array of SVtx: sv[0]=(net[0],idx[0]) to vertices sv[k], k=1,...,n-1. */
    471:   PetscCall(PetscMalloc1(nta, &svtx));
    472:   for (nsv = 0; nsv < nta; nsv++) {
    473:     /* for a single svtx, put shared vertices in ascending order of gidx */
    474:     PetscCall(PetscHMapIGetSize(svtas[nsv], &n));
    475:     PetscCall(PetscCalloc1(2 * n, &sv));
    476:     PetscCall(PetscMalloc3(n, &gidx_tmp, n, &net_tmp, n, &idx_tmp));
    477:     svtx[nsv].sv   = sv;
    478:     svtx[nsv].n    = n;
    479:     svtx[nsv].gidx = network->cloneshared->NVertices; /* initialization */

    481:     PetscHashIterBegin(svtas[nsv], ppos);
    482:     for (k = 0; k < n; k++) { /* gidx is sorted in ascending order */
    483:       PetscHashIterGetKey(svtas[nsv], ppos, gidx);
    484:       PetscHashIterGetVal(svtas[nsv], ppos, i);
    485:       PetscHashIterNext(svtas[nsv], ppos);
    486:       gidx--;
    487:       i--;
    488:       j           = ta2sv[nsv][i];    /* maps i to index of sedgelist */
    489:       net_tmp[k]  = sedgelist[j];     /* subnet number */
    490:       idx_tmp[k]  = sedgelist[j + 1]; /* index on the subnet */
    491:       gidx_tmp[k] = gidx;             /* gidx in un-merged dmnetwork */
    492:     }

    494:     /* current implementation requires sv[]=[net,idx] in ascending order of its gidx in un-merged dmnetwork */
    495:     PetscCall(PetscSortIntWithArrayPair(n, gidx_tmp, net_tmp, idx_tmp));
    496:     svtx[nsv].gidx = gidx_tmp[0]; /* = min(gidx) */
    497:     for (k = 0; k < n; k++) {
    498:       sv[2 * k]     = net_tmp[k];
    499:       sv[2 * k + 1] = idx_tmp[k];
    500:     }
    501:     PetscCall(PetscFree3(gidx_tmp, net_tmp, idx_tmp));

    503:     /* Setup svtable for query shared vertices */
    504:     PetscCall(PetscHMapISet(network->cloneshared->svtable, svtx[nsv].gidx + 1, nsv + 1));
    505:   }

    507:   for (j = 0; j < nta; j++) {
    508:     PetscCall(PetscHMapIDestroy(svtas + j));
    509:     PetscCall(PetscFree(ta2sv[j]));
    510:   }
    511:   PetscCall(PetscFree3(svtas, tdata, ta2sv));

    513:   network->cloneshared->Nsvtx = nta;
    514:   network->cloneshared->svtx  = svtx;
    515:   PetscFunctionReturn(PETSC_SUCCESS);
    516: }

    518: /*
    519:   GetEdgelist_Coupling - Get an integrated edgelist for dmplex from user-provided subnet[].edgelist when subnets are coupled by shared vertices

    521:   Input Parameters:
    522: . dm - the dmnetwork object

    524:    Output Parameters:
    525: +  edges - the integrated edgelist for dmplex
    526: -  nmerged_ptr - num of vertices being merged
    527: */
    528: static PetscErrorCode GetEdgelist_Coupling(DM dm, PetscInt *edges, PetscInt *nmerged_ptr)
    529: {
    530:   MPI_Comm    comm;
    531:   PetscMPIInt size, rank;
    532:   DM_Network *network = (DM_Network *)dm->data;
    533:   PetscInt    i, j, ctr, np;
    534:   PetscInt   *vidxlTog, Nsv, Nsubnet = network->cloneshared->Nsubnet;
    535:   PetscInt   *sedgelist = network->cloneshared->sedgelist, vrange;
    536:   PetscInt    net, idx, gidx, nmerged, gidx_from, net_from, sv_idx;
    537:   SVtxType    svtype = SVNONE;
    538:   SVtx       *svtx;

    540:   PetscFunctionBegin;
    541:   PetscCall(PetscObjectGetComm((PetscObject)dm, &comm));
    542:   PetscCallMPI(MPI_Comm_rank(comm, &rank));
    543:   PetscCallMPI(MPI_Comm_size(comm, &size));

    545:   /* (1) Create global svtx[] from sedgelist */
    546:   /* --------------------------------------- */
    547:   PetscCall(SharedVtxCreate(dm, network->cloneshared->Nsvtx, sedgelist));
    548:   Nsv  = network->cloneshared->Nsvtx;
    549:   svtx = network->cloneshared->svtx;

    551:   /* (2) Merge shared vto vertices to their vfrom vertex with same global vertex index (gidx) */
    552:   /* --------------------------------------------------------------------------------------- */
    553:   /* (2.1) compute vrage[rank]: global index of 1st local vertex in proc[rank] */
    554:   PetscCall(PetscMalloc1(network->cloneshared->nVertices, &vidxlTog));

    556:   PetscCallMPI(MPI_Scan(&network->cloneshared->nVertices, &vrange, 1, MPIU_INT, MPI_SUM, comm));
    557:   vrange -= network->cloneshared->nVertices;

    559:   /* (2.2) Create vidxlTog: maps UN-MERGED local vertex index i to global index gidx (plex, excluding ghost vertices) */
    560:   i                           = 0;
    561:   gidx                        = 0;
    562:   nmerged                     = 0; /* local num of merged vertices */
    563:   network->cloneshared->nsvtx = 0; /* local num of SVtx structs, including ghosts */
    564:   for (net = 0; net < Nsubnet; net++) {
    565:     for (idx = 0; idx < network->cloneshared->subnet[net].Nvtx; idx++) { /* Note: global subnet[net].Nvtx */
    566:       PetscCall(VtxGetInfo(Nsv, svtx, net, idx, &gidx_from, &svtype, &sv_idx));
    567:       if (svtype == SVTO) {
    568:         if (network->cloneshared->subnet[net].nvtx) { /* this proc owns sv_to */
    569:           net_from = svtx[sv_idx].sv[0];              /* subnet number of its shared vertex */
    570:           if (network->cloneshared->subnet[net_from].nvtx == 0) {
    571:             /* this proc does not own v_from, thus a ghost local vertex */
    572:             network->cloneshared->nsvtx++;
    573:           }
    574:           vidxlTog[i++] = gidx_from; /* gidx before merging! Bug??? */
    575:           nmerged++;                 /* a shared vertex -- merged */
    576:         }
    577:       } else {
    578:         if (svtype == SVFROM && network->cloneshared->subnet[net].nvtx) {
    579:           /* this proc owns this v_from, a new local shared vertex */
    580:           network->cloneshared->nsvtx++;
    581:         }
    582:         if (network->cloneshared->subnet[net].nvtx) vidxlTog[i++] = gidx;
    583:         gidx++;
    584:       }
    585:     }
    586:   }
    587:   PetscAssert(i == network->cloneshared->nVertices, PETSC_COMM_SELF, PETSC_ERR_ARG_NULL, "%" PetscInt_FMT " != %" PetscInt_FMT " nVertices", i, network->cloneshared->nVertices);

    589:   /* (2.3) Shared vertices in the subnetworks are merged, update global NVertices: np = sum(local nmerged) */
    590:   PetscCallMPI(MPIU_Allreduce(&nmerged, &np, 1, MPIU_INT, MPI_SUM, comm));
    591:   network->cloneshared->NVertices -= np;

    593:   ctr = 0;
    594:   for (net = 0; net < Nsubnet; net++) {
    595:     for (j = 0; j < network->cloneshared->subnet[net].nedge; j++) {
    596:       /* vfrom: */
    597:       i              = network->cloneshared->subnet[net].edgelist[2 * j] + (network->cloneshared->subnet[net].vStart - vrange);
    598:       edges[2 * ctr] = vidxlTog[i];

    600:       /* vto */
    601:       i                  = network->cloneshared->subnet[net].edgelist[2 * j + 1] + (network->cloneshared->subnet[net].vStart - vrange);
    602:       edges[2 * ctr + 1] = vidxlTog[i];
    603:       ctr++;
    604:     }
    605:   }
    606:   PetscCall(PetscFree(vidxlTog));
    607:   PetscCall(PetscFree(sedgelist)); /* created in DMNetworkAddSharedVertices() */

    609:   *nmerged_ptr = nmerged;
    610:   PetscFunctionReturn(PETSC_SUCCESS);
    611: }

    613: PetscErrorCode DMNetworkInitializeNonTopological(DM dm)
    614: {
    615:   DM_Network *network = (DM_Network *)dm->data;
    616:   PetscInt    p, pStart = network->cloneshared->pStart, pEnd = network->cloneshared->pEnd;
    617:   MPI_Comm    comm;

    619:   PetscFunctionBegin;
    620:   PetscCall(PetscObjectGetComm((PetscObject)dm, &comm));

    622:   PetscCall(PetscSectionCreate(comm, &network->DataSection));
    623:   PetscCall(PetscSectionCreate(comm, &network->DofSection));
    624:   PetscCall(PetscSectionSetChart(network->DataSection, pStart, pEnd));
    625:   PetscCall(PetscSectionSetChart(network->DofSection, pStart, pEnd));

    627:   PetscCall(DMNetworkInitializeHeaderComponentData(dm));

    629:   for (p = 0; p < pEnd - pStart; p++) {
    630:     network->header[p].ndata           = 0;
    631:     network->header[p].offset[0]       = 0;
    632:     network->header[p].offsetvarrel[0] = 0;
    633:     PetscCall(PetscSectionAddDof(network->DataSection, p, network->header[p].hsize));
    634:   }
    635:   PetscFunctionReturn(PETSC_SUCCESS);
    636: }

    638: /*@
    639:   DMNetworkLayoutSetUp - Sets up the bare layout (graph) for the network

    641:   Not Collective

    643:   Input Parameter:
    644: . dm - the `DMNETWORK` object

    646:   Level: beginner

    648:   Notes:
    649:   This routine should be called after the network sizes and edgelists have been provided. It creates
    650:   the bare layout of the network and sets up the network to begin insertion of components.

    652:   All the components should be registered before calling this routine.

    654: .seealso: `DM`, `DMNETWORK`, `DMNetworkSetNumSubNetworks()`, `DMNetworkAddSubnetwork()`
    655: @*/
    656: PetscErrorCode DMNetworkLayoutSetUp(DM dm)
    657: {
    658:   DM_Network     *network = (DM_Network *)dm->data;
    659:   PetscInt        i, j, ctr, Nsubnet = network->cloneshared->Nsubnet, np, *edges, *subnetvtx, *subnetedge, e, v, vfrom, vto, net, globaledgeoff;
    660:   const PetscInt *cone;
    661:   MPI_Comm        comm;
    662:   PetscMPIInt     size;
    663:   PetscSection    sectiong;
    664:   PetscInt        nmerged = 0;

    666:   PetscFunctionBegin;
    667:   PetscCall(PetscLogEventBegin(DMNetwork_LayoutSetUp, dm, 0, 0, 0));
    668:   PetscCheck(network->cloneshared->nsubnet == Nsubnet, PETSC_COMM_SELF, PETSC_ERR_ARG_INCOMP, "Must call DMNetworkAddSubnetwork() %" PetscInt_FMT " times", Nsubnet);

    670:   /* This implementation requires user input each subnet by a single processor when Nsubnet>1, thus subnet[net].nvtx=subnet[net].Nvtx when net>0 */
    671:   for (net = 1; net < Nsubnet; net++) {
    672:     if (network->cloneshared->subnet[net].nvtx)
    673:       PetscCheck(network->cloneshared->subnet[net].nvtx == network->cloneshared->subnet[net].Nvtx, PETSC_COMM_SELF, PETSC_ERR_SUP, "subnetwork %" PetscInt_FMT " local num of vertices %" PetscInt_FMT " != %" PetscInt_FMT " global num", net,
    674:                  network->cloneshared->subnet[net].nvtx, network->cloneshared->subnet[net].Nvtx);
    675:   }

    677:   PetscCall(PetscObjectGetComm((PetscObject)dm, &comm));
    678:   PetscCallMPI(MPI_Comm_size(comm, &size));

    680:   /* Create LOCAL edgelist in global vertex ordering for the network by concatenating local input edgelists of the subnetworks */
    681:   PetscCall(PetscCalloc1(2 * network->cloneshared->nEdges, &edges));

    683:   if (network->cloneshared->Nsvtx) { /* subnetworks are coupled via shared vertices */
    684:     PetscCall(GetEdgelist_Coupling(dm, edges, &nmerged));
    685:   } else { /* subnetworks are not coupled */
    686:     /* Create a 0-size svtable for query shared vertices */
    687:     PetscCall(PetscHMapICreate(&network->cloneshared->svtable));
    688:     ctr = 0;
    689:     for (i = 0; i < Nsubnet; i++) {
    690:       for (j = 0; j < network->cloneshared->subnet[i].nedge; j++) {
    691:         edges[2 * ctr]     = network->cloneshared->subnet[i].vStart + network->cloneshared->subnet[i].edgelist[2 * j];
    692:         edges[2 * ctr + 1] = network->cloneshared->subnet[i].vStart + network->cloneshared->subnet[i].edgelist[2 * j + 1];
    693:         ctr++;
    694:       }
    695:     }
    696:   }

    698:   /* Create network->plex; One dimensional network, numCorners=2 */
    699:   PetscCall(DMCreate(comm, &network->plex));
    700:   PetscCall(DMSetType(network->plex, DMPLEX));
    701:   PetscCall(DMSetDimension(network->plex, 1));

    703:   if (size == 1) PetscCall(DMPlexBuildFromCellList(network->plex, network->cloneshared->nEdges, PETSC_DECIDE, 2, edges));
    704:   else PetscCall(DMPlexBuildFromCellListParallel(network->plex, network->cloneshared->nEdges, PETSC_DECIDE, PETSC_DECIDE, 2, edges, NULL, NULL));

    706:   PetscCall(DMPlexGetChart(network->plex, &network->cloneshared->pStart, &network->cloneshared->pEnd));
    707:   PetscCall(DMPlexGetHeightStratum(network->plex, 0, &network->cloneshared->eStart, &network->cloneshared->eEnd));
    708:   PetscCall(DMPlexGetHeightStratum(network->plex, 1, &network->cloneshared->vStart, &network->cloneshared->vEnd));
    709:   np = network->cloneshared->pEnd - network->cloneshared->pStart;
    710:   PetscCall(PetscCalloc2(np, &network->header, np, &network->cvalue));

    712:   /* Create edge and vertex arrays for the subnetworks
    713:      This implementation assumes that DMNetwork reads
    714:      (1) a single subnetwork in parallel; or
    715:      (2) n subnetworks using n processors, one subnetwork/processor.
    716:   */
    717:   PetscCall(PetscCalloc2(network->cloneshared->nEdges, &subnetedge, network->cloneshared->nVertices + network->cloneshared->nsvtx, &subnetvtx)); /* Maps local edge/vertex to local subnetwork's edge/vertex */
    718:   network->cloneshared->subnetedge = subnetedge;
    719:   network->cloneshared->subnetvtx  = subnetvtx;
    720:   for (j = 0; j < Nsubnet; j++) {
    721:     network->cloneshared->subnet[j].edges = subnetedge;
    722:     subnetedge                            = PetscSafePointerPlusOffset(subnetedge, network->cloneshared->subnet[j].nedge);

    724:     network->cloneshared->subnet[j].vertices = subnetvtx;
    725:     subnetvtx                                = PetscSafePointerPlusOffset(subnetvtx, network->cloneshared->subnet[j].nvtx);
    726:   }
    727:   network->cloneshared->svertices = subnetvtx;

    729:   /* Get edge ownership */
    730:   np = network->cloneshared->eEnd - network->cloneshared->eStart;
    731:   PetscCallMPI(MPI_Scan(&np, &globaledgeoff, 1, MPIU_INT, MPI_SUM, comm));
    732:   globaledgeoff -= np;

    734:   /* Setup local edge and vertex arrays for subnetworks */
    735:   e = 0;
    736:   for (i = 0; i < Nsubnet; i++) {
    737:     ctr = 0;
    738:     for (j = 0; j < network->cloneshared->subnet[i].nedge; j++) {
    739:       /* edge e */
    740:       network->header[e].index                 = e + globaledgeoff; /* Global edge index */
    741:       network->header[e].subnetid              = i;
    742:       network->cloneshared->subnet[i].edges[j] = e;

    744:       /* connected vertices */
    745:       PetscCall(DMPlexGetCone(network->plex, e, &cone));

    747:       /* vertex cone[0] */
    748:       v                           = cone[0];
    749:       network->header[v].index    = edges[2 * e]; /* Global vertex index */
    750:       network->header[v].subnetid = i;            /* Subnetwork id */
    751:       if (Nsubnet == 1) {
    752:         network->cloneshared->subnet[i].vertices[v - network->cloneshared->vStart] = v; /* user's subnet[].idx = petsc's v */
    753:       } else {
    754:         vfrom                                           = network->cloneshared->subnet[i].edgelist[2 * ctr]; /* =subnet[i].idx, Global index! */
    755:         network->cloneshared->subnet[i].vertices[vfrom] = v;                                                 /* user's subnet[].dix = petsc's v */
    756:       }

    758:       /* vertex cone[1] */
    759:       v                           = cone[1];
    760:       network->header[v].index    = edges[2 * e + 1]; /* Global vertex index */
    761:       network->header[v].subnetid = i;                /* Subnetwork id */
    762:       if (Nsubnet == 1) {
    763:         network->cloneshared->subnet[i].vertices[v - network->cloneshared->vStart] = v; /* user's subnet[].idx = petsc's v */
    764:       } else {
    765:         vto                                           = network->cloneshared->subnet[i].edgelist[2 * ctr + 1]; /* =subnet[i].idx, Global index! */
    766:         network->cloneshared->subnet[i].vertices[vto] = v;                                                     /* user's subnet[].dix = petsc's v */
    767:       }

    769:       e++;
    770:       ctr++;
    771:     }
    772:   }
    773:   PetscCall(PetscFree(edges));

    775:   /* Set local vertex array for the subnetworks */
    776:   j = 0;
    777:   for (v = network->cloneshared->vStart; v < network->cloneshared->vEnd; v++) {
    778:     /* local shared vertex */
    779:     PetscCall(PetscHMapIGetWithDefault(network->cloneshared->svtable, network->header[v].index + 1, 0, &i));
    780:     if (i) network->cloneshared->svertices[j++] = v;
    781:   }

    783:   /* Create a global section to be used by DMNetworkIsGhostVertex() which is a non-collective routine */
    784:   /* see snes_tutorials_network-ex1_4 */
    785:   PetscCall(DMGetGlobalSection(network->plex, &sectiong));
    786:   /* Initialize non-topological data structures  */
    787:   PetscCall(DMNetworkInitializeNonTopological(dm));
    788:   PetscCall(PetscLogEventEnd(DMNetwork_LayoutSetUp, dm, 0, 0, 0));
    789:   PetscFunctionReturn(PETSC_SUCCESS);
    790: }

    792: /*@C
    793:   DMNetworkGetSubnetwork - Returns the information about a requested subnetwork

    795:   Not Collective

    797:   Input Parameters:
    798: + dm     - the `DMNETWORK` object
    799: - netnum - the global index of the subnetwork

    801:   Output Parameters:
    802: + nv   - number of vertices (local)
    803: . ne   - number of edges (local)
    804: . vtx  - local vertices of the subnetwork
    805: - edge - local edges of the subnetwork

    807:   Level: intermediate

    809:   Notes:
    810:   Pass `NULL` for any output argument that is not needed.

    812:   Cannot call this routine before `DMNetworkLayoutSetup()`

    814:   The local vertices returned on each rank are determined by `DMNETWORK`. The user does not have any control over what vertices are local.

    816: .seealso: `DM`, `DMNETWORK`, `DMNetworkCreate()`, `DMNetworkAddSubnetwork()`, `DMNetworkLayoutSetUp()`
    817: @*/
    818: PetscErrorCode DMNetworkGetSubnetwork(DM dm, PetscInt netnum, PeOp PetscInt *nv, PeOp PetscInt *ne, PeOp const PetscInt *vtx[], PeOp const PetscInt *edge[])
    819: {
    820:   DM_Network *network = (DM_Network *)dm->data;

    822:   PetscFunctionBegin;
    823:   PetscCheck(netnum < network->cloneshared->Nsubnet, PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Subnet index %" PetscInt_FMT " exceeds the num of subnets %" PetscInt_FMT, netnum, network->cloneshared->Nsubnet);
    824:   if (nv) *nv = network->cloneshared->subnet[netnum].nvtx;
    825:   if (ne) *ne = network->cloneshared->subnet[netnum].nedge;
    826:   if (vtx) *vtx = network->cloneshared->subnet[netnum].vertices;
    827:   if (edge) *edge = network->cloneshared->subnet[netnum].edges;
    828:   PetscFunctionReturn(PETSC_SUCCESS);
    829: }

    831: /*@
    832:   DMNetworkAddSharedVertices - Add shared vertices that connect two given subnetworks

    834:   Collective

    836:   Input Parameters:
    837: + dm      - the `DMNETWORK` object
    838: . anetnum - first subnetwork global numbering returned by `DMNetworkAddSubnetwork()`
    839: . bnetnum - second subnetwork global numbering returned by `DMNetworkAddSubnetwork()`
    840: . nsvtx   - number of vertices that are shared by the two subnetworks
    841: . asvtx   - vertex index in the first subnetwork
    842: - bsvtx   - vertex index in the second subnetwork

    844:   Level: beginner

    846: .seealso: `DM`, `DMNETWORK`, `DMNetworkCreate()`, `DMNetworkAddSubnetwork()`, `DMNetworkGetSharedVertices()`
    847: @*/
    848: PetscErrorCode DMNetworkAddSharedVertices(DM dm, PetscInt anetnum, PetscInt bnetnum, PetscInt nsvtx, PetscInt asvtx[], PetscInt bsvtx[])
    849: {
    850:   DM_Network *network = (DM_Network *)dm->data;
    851:   PetscInt    i, nsubnet = network->cloneshared->Nsubnet, *sedgelist, Nsvtx = network->cloneshared->Nsvtx;

    853:   PetscFunctionBegin;
    854:   PetscCheck(anetnum != bnetnum, PetscObjectComm((PetscObject)dm), PETSC_ERR_USER, "Subnetworks must have different netnum");
    855:   PetscCheck(anetnum >= 0 && bnetnum >= 0, PetscObjectComm((PetscObject)dm), PETSC_ERR_USER, "netnum cannot be negative");
    856:   if (!Nsvtx) {
    857:     /* allocate network->sedgelist to hold at most 2*nsubnet pairs of shared vertices */
    858:     PetscCall(PetscMalloc1(2 * 4 * nsubnet, &network->cloneshared->sedgelist));
    859:   }

    861:   sedgelist = network->cloneshared->sedgelist;
    862:   for (i = 0; i < nsvtx; i++) {
    863:     sedgelist[4 * Nsvtx]     = anetnum;
    864:     sedgelist[4 * Nsvtx + 1] = asvtx[i];
    865:     sedgelist[4 * Nsvtx + 2] = bnetnum;
    866:     sedgelist[4 * Nsvtx + 3] = bsvtx[i];
    867:     Nsvtx++;
    868:   }
    869:   PetscCheck(Nsvtx <= 2 * nsubnet, PETSC_COMM_SELF, PETSC_ERR_SUP, "allocate more space for coupling edgelist");
    870:   network->cloneshared->Nsvtx = Nsvtx;
    871:   PetscFunctionReturn(PETSC_SUCCESS);
    872: }

    874: /*@C
    875:   DMNetworkGetSharedVertices - Returns the info for the shared vertices

    877:   Not Collective

    879:   Input Parameter:
    880: . dm - the `DMNETWORK` object

    882:   Output Parameters:
    883: + nsv  - number of local shared vertices, pass `NULL` if not needed
    884: - svtx - local shared vertices, pass `NULL` if not needed

    886:   Level: intermediate

    888:   Note:
    889:   Cannot call this routine before `DMNetworkLayoutSetup()`

    891: .seealso: `DM`, `DMNETWORK`, `DMNetworkGetSubnetwork()`, `DMNetworkLayoutSetUp()`, `DMNetworkAddSharedVertices()`
    892: @*/
    893: PetscErrorCode DMNetworkGetSharedVertices(DM dm, PeOp PetscInt *nsv, PeOp const PetscInt *svtx[])
    894: {
    895:   DM_Network *net = (DM_Network *)dm->data;

    897:   PetscFunctionBegin;
    899:   if (nsv) *nsv = net->cloneshared->nsvtx;
    900:   if (svtx) *svtx = net->cloneshared->svertices;
    901:   PetscFunctionReturn(PETSC_SUCCESS);
    902: }

    904: /*@C
    905:   DMNetworkRegisterComponent - Registers the network component

    907:   Logically Collective

    909:   Input Parameters:
    910: + dm   - the `DMNETWORK` object
    911: . name - the component name
    912: - size - the storage size in bytes for this component data

    914:   Output Parameter:
    915: . key - an integer key that defines the component

    917:   Level: beginner

    919:   Note:
    920:   This routine should be called by all processors before calling `DMNetworkLayoutSetup()`.

    922: .seealso: `DM`, `DMNETWORK`, `DMNetworkCreate()`, `DMNetworkLayoutSetUp()`
    923: @*/
    924: PetscErrorCode DMNetworkRegisterComponent(DM dm, const char *name, size_t size, PetscInt *key)
    925: {
    926:   DM_Network         *network   = (DM_Network *)dm->data;
    927:   DMNetworkComponent *component = NULL, *newcomponent = NULL;
    928:   PetscBool           flg = PETSC_FALSE;
    929:   PetscInt            i;

    931:   PetscFunctionBegin;
    932:   if (!network->component) PetscCall(PetscCalloc1(network->max_comps_registered, &network->component));

    934:   for (i = 0; i < network->ncomponent; i++) {
    935:     PetscCall(PetscStrcmp(network->component[i].name, name, &flg));
    936:     if (flg) {
    937:       *key = i;
    938:       PetscFunctionReturn(PETSC_SUCCESS);
    939:     }
    940:   }

    942:   if (network->ncomponent == network->max_comps_registered) {
    943:     /* Reached max allowed so resize component */
    944:     network->max_comps_registered += 2;
    945:     PetscCall(PetscCalloc1(network->max_comps_registered, &newcomponent));
    946:     /* Copy over the previous component info */
    947:     for (i = 0; i < network->ncomponent; i++) {
    948:       PetscCall(PetscStrncpy(newcomponent[i].name, network->component[i].name, sizeof(newcomponent[i].name)));
    949:       newcomponent[i].size = network->component[i].size;
    950:     }
    951:     /* Free old one */
    952:     PetscCall(PetscFree(network->component));
    953:     /* Update pointer */
    954:     network->component = newcomponent;
    955:   }

    957:   component = &network->component[network->ncomponent];

    959:   PetscCall(PetscStrncpy(component->name, name, sizeof(component->name)));
    960:   PetscCheck((size % sizeof(DMNetworkComponentGenericDataType)) == 0, PETSC_COMM_SELF, PETSC_ERR_ARG_SIZ, "Size of datatype must be divisible by sizeof(DMNetworkComponentGenericDataType)");
    961:   PetscCall(PetscIntCast(size / sizeof(DMNetworkComponentGenericDataType), &component->size));
    962:   *key = network->ncomponent;
    963:   network->ncomponent++;
    964:   PetscFunctionReturn(PETSC_SUCCESS);
    965: }

    967: /*@
    968:   DMNetworkGetNumVertices - Get the local and global number of vertices for the entire network.

    970:   Not Collective

    972:   Input Parameter:
    973: . dm - the `DMNETWORK` object

    975:   Output Parameters:
    976: + nVertices - the local number of vertices, pass `NULL` if not needed
    977: - NVertices - the global number of vertices, pass `NULL` if not needed

    979:   Level: beginner

    981: .seealso: `DM`, `DMNETWORK`, `DMNetworkGetNumEdges()`
    982: @*/
    983: PetscErrorCode DMNetworkGetNumVertices(DM dm, PeOp PetscInt *nVertices, PeOp PetscInt *NVertices)
    984: {
    985:   DM_Network *network = (DM_Network *)dm->data;

    987:   PetscFunctionBegin;
    989:   if (nVertices) {
    990:     PetscAssertPointer(nVertices, 2);
    991:     *nVertices = network->cloneshared->nVertices;
    992:   }
    993:   if (NVertices) {
    994:     PetscAssertPointer(NVertices, 3);
    995:     *NVertices = network->cloneshared->NVertices;
    996:   }
    997:   PetscFunctionReturn(PETSC_SUCCESS);
    998: }

    1000: /*@
    1001:   DMNetworkGetNumEdges - Get the local and global number of edges for the entire network.

    1003:   Not Collective

    1005:   Input Parameter:
    1006: . dm - the `DMNETWORK` object

    1008:   Output Parameters:
    1009: + nEdges - the local number of edges, pass `NULL` if not needed
    1010: - NEdges - the global number of edges, pass `NULL` if not needed

    1012:   Level: beginner

    1014: .seealso: `DM`, `DMNETWORK`, `DMNetworkGetNumVertices()`
    1015: @*/
    1016: PetscErrorCode DMNetworkGetNumEdges(DM dm, PeOp PetscInt *nEdges, PeOp PetscInt *NEdges)
    1017: {
    1018:   DM_Network *network = (DM_Network *)dm->data;

    1020:   PetscFunctionBegin;
    1022:   if (nEdges) {
    1023:     PetscAssertPointer(nEdges, 2);
    1024:     *nEdges = network->cloneshared->nEdges;
    1025:   }
    1026:   if (NEdges) {
    1027:     PetscAssertPointer(NEdges, 3);
    1028:     *NEdges = network->cloneshared->NEdges;
    1029:   }
    1030:   PetscFunctionReturn(PETSC_SUCCESS);
    1031: }

    1033: /*@
    1034:   DMNetworkGetVertexRange - Get the bounds [start, end) for the local vertices

    1036:   Not Collective

    1038:   Input Parameter:
    1039: . dm - the `DMNETWORK` object

    1041:   Output Parameters:
    1042: + vStart - the first vertex point, pass `NULL` if not needed
    1043: - vEnd   - one beyond the last vertex point, pass `NULL` if not needed

    1045:   Level: beginner

    1047: .seealso: `DM`, `DMNETWORK`, `DMNetworkGetEdgeRange()`
    1048: @*/
    1049: PetscErrorCode DMNetworkGetVertexRange(DM dm, PeOp PetscInt *vStart, PeOp PetscInt *vEnd)
    1050: {
    1051:   DM_Network *network = (DM_Network *)dm->data;

    1053:   PetscFunctionBegin;
    1054:   if (vStart) *vStart = network->cloneshared->vStart;
    1055:   if (vEnd) *vEnd = network->cloneshared->vEnd;
    1056:   PetscFunctionReturn(PETSC_SUCCESS);
    1057: }

    1059: /*@
    1060:   DMNetworkGetEdgeRange - Get the bounds [start, end) for the local edges

    1062:   Not Collective

    1064:   Input Parameter:
    1065: . dm - the `DMNETWORK` object

    1067:   Output Parameters:
    1068: + eStart - The first edge point, pass `NULL` if not needed
    1069: - eEnd   - One beyond the last edge point, pass `NULL` if not needed

    1071:   Level: beginner

    1073: .seealso: `DM`, `DMNETWORK`, `DMNetworkGetVertexRange()`
    1074: @*/
    1075: PetscErrorCode DMNetworkGetEdgeRange(DM dm, PeOp PetscInt *eStart, PeOp PetscInt *eEnd)
    1076: {
    1077:   DM_Network *network = (DM_Network *)dm->data;

    1079:   PetscFunctionBegin;
    1081:   if (eStart) *eStart = network->cloneshared->eStart;
    1082:   if (eEnd) *eEnd = network->cloneshared->eEnd;
    1083:   PetscFunctionReturn(PETSC_SUCCESS);
    1084: }

    1086: PetscErrorCode DMNetworkGetIndex(DM dm, PetscInt p, PetscInt *index)
    1087: {
    1088:   DM_Network *network = (DM_Network *)dm->data;

    1090:   PetscFunctionBegin;
    1091:   if (network->header) {
    1092:     *index = network->header[p].index;
    1093:   } else {
    1094:     PetscInt                 offsetp;
    1095:     DMNetworkComponentHeader header;

    1097:     PetscCall(PetscSectionGetOffset(network->DataSection, p, &offsetp));
    1098:     header = (DMNetworkComponentHeader)(network->componentdataarray + offsetp);
    1099:     *index = header->index;
    1100:   }
    1101:   PetscFunctionReturn(PETSC_SUCCESS);
    1102: }

    1104: PetscErrorCode DMNetworkGetSubnetID(DM dm, PetscInt p, PetscInt *subnetid)
    1105: {
    1106:   DM_Network *network = (DM_Network *)dm->data;

    1108:   PetscFunctionBegin;
    1109:   if (network->header) {
    1110:     *subnetid = network->header[p].subnetid;
    1111:   } else {
    1112:     PetscInt                 offsetp;
    1113:     DMNetworkComponentHeader header;

    1115:     PetscCall(PetscSectionGetOffset(network->DataSection, p, &offsetp));
    1116:     header    = (DMNetworkComponentHeader)(network->componentdataarray + offsetp);
    1117:     *subnetid = header->subnetid;
    1118:   }
    1119:   PetscFunctionReturn(PETSC_SUCCESS);
    1120: }

    1122: /*@
    1123:   DMNetworkGetGlobalEdgeIndex - Get the global numbering for the edge on the network

    1125:   Not Collective

    1127:   Input Parameters:
    1128: + dm - `DMNETWORK` object
    1129: - p  - edge point

    1131:   Output Parameter:
    1132: . index - the global numbering for the edge

    1134:   Level: intermediate

    1136: .seealso: `DM`, `DMNETWORK`, `DMNetworkGetGlobalVertexIndex()`
    1137: @*/
    1138: PetscErrorCode DMNetworkGetGlobalEdgeIndex(DM dm, PetscInt p, PetscInt *index)
    1139: {
    1140:   PetscFunctionBegin;
    1141:   PetscCall(DMNetworkGetIndex(dm, p, index));
    1142:   PetscFunctionReturn(PETSC_SUCCESS);
    1143: }

    1145: /*@
    1146:   DMNetworkGetGlobalVertexIndex - Get the global numbering for the vertex on the network

    1148:   Not Collective

    1150:   Input Parameters:
    1151: + dm - `DMNETWORK` object
    1152: - p  - vertex point

    1154:   Output Parameter:
    1155: . index - the global numbering for the vertex

    1157:   Level: intermediate

    1159: .seealso: `DM`, `DMNETWORK`, `DMNetworkGetGlobalEdgeIndex()`, `DMNetworkGetLocalVertexIndex()`
    1160: @*/
    1161: PetscErrorCode DMNetworkGetGlobalVertexIndex(DM dm, PetscInt p, PetscInt *index)
    1162: {
    1163:   PetscFunctionBegin;
    1164:   PetscCall(DMNetworkGetIndex(dm, p, index));
    1165:   PetscFunctionReturn(PETSC_SUCCESS);
    1166: }

    1168: /*@
    1169:   DMNetworkGetNumComponents - Get the number of components at a vertex/edge

    1171:   Not Collective

    1173:   Input Parameters:
    1174: + dm - the `DMNETWORK` object
    1175: - p  - vertex/edge point

    1177:   Output Parameter:
    1178: . numcomponents - Number of components at the vertex/edge

    1180:   Level: beginner

    1182: .seealso: `DM`, `DMNETWORK`, `DMNetworkRegisterComponent()`, `DMNetworkAddComponent()`
    1183: @*/
    1184: PetscErrorCode DMNetworkGetNumComponents(DM dm, PetscInt p, PetscInt *numcomponents)
    1185: {
    1186:   PetscInt    offset;
    1187:   DM_Network *network = (DM_Network *)dm->data;

    1189:   PetscFunctionBegin;
    1190:   PetscCall(PetscSectionGetOffset(network->DataSection, p, &offset));
    1191:   *numcomponents = ((DMNetworkComponentHeader)(network->componentdataarray + offset))->ndata;
    1192:   PetscFunctionReturn(PETSC_SUCCESS);
    1193: }

    1195: /*@
    1196:   DMNetworkGetLocalVecOffset - Get the offset for accessing the variables associated with a component at the given vertex/edge from the local vector

    1198:   Not Collective

    1200:   Input Parameters:
    1201: + dm      - the `DMNETWORK` object
    1202: . p       - the edge or vertex point
    1203: - compnum - component number; use ALL_COMPONENTS if no specific component is requested

    1205:   Output Parameter:
    1206: . offset - the local offset

    1208:   Level: intermediate

    1210:   Notes:
    1211:   These offsets can be passed to `MatSetValuesLocal()` for matrices obtained with `DMCreateMatrix()`.

    1213:   For vectors obtained with `DMCreateLocalVector()` the offsets can be used with `VecSetValues()`.

    1215:   For vectors obtained with `DMCreateLocalVector()` and the array obtained with `VecGetArray`(vec,&array) you can access or set
    1216:   the vector values with array[offset].

    1218:   For vectors obtained with `DMCreateGlobalVector()` the offsets can be used with `VecSetValuesLocal()`.

    1220: .seealso: `DM`, `DMNETWORK`, `DMGetLocalVector()`, `DMNetworkGetComponent()`, `DMNetworkGetGlobalVecOffset()`, `DMCreateGlobalVector()`, `VecGetArray()`, `VecSetValuesLocal()`, `MatSetValuesLocal()`
    1221: @*/
    1222: PetscErrorCode DMNetworkGetLocalVecOffset(DM dm, PetscInt p, PetscInt compnum, PetscInt *offset)
    1223: {
    1224:   DM_Network              *network = (DM_Network *)dm->data;
    1225:   PetscInt                 offsetp, offsetd;
    1226:   DMNetworkComponentHeader header;

    1228:   PetscFunctionBegin;
    1229:   PetscCall(PetscSectionGetOffset(network->plex->localSection, p, &offsetp));
    1230:   if (compnum == ALL_COMPONENTS) {
    1231:     *offset = offsetp;
    1232:     PetscFunctionReturn(PETSC_SUCCESS);
    1233:   }

    1235:   PetscCall(PetscSectionGetOffset(network->DataSection, p, &offsetd));
    1236:   header  = (DMNetworkComponentHeader)(network->componentdataarray + offsetd);
    1237:   *offset = offsetp + header->offsetvarrel[compnum];
    1238:   PetscFunctionReturn(PETSC_SUCCESS);
    1239: }

    1241: /*@
    1242:   DMNetworkGetGlobalVecOffset - Get the global offset for accessing the variables associated with a component for the given vertex/edge from the global vector

    1244:   Not Collective

    1246:   Input Parameters:
    1247: + dm      - the `DMNETWORK` object
    1248: . p       - the edge or vertex point
    1249: - compnum - component number; use ALL_COMPONENTS if no specific component is requested

    1251:   Output Parameter:
    1252: . offsetg - the global offset

    1254:   Level: intermediate

    1256:   Notes:
    1257:   These offsets can be passed to `MatSetValues()` for matrices obtained with `DMCreateMatrix()`.

    1259:   For vectors obtained with `DMCreateGlobalVector()` the offsets can be used with `VecSetValues()`.

    1261:   For vectors obtained with `DMCreateGlobalVector()` and the array obtained with `VecGetArray`(vec,&array) you can access or set
    1262:   the vector values with array[offset - rstart] where restart is obtained with `VecGetOwnershipRange`(v,&rstart,`NULL`);

    1264: .seealso: `DM`, `DMNETWORK`, `DMNetworkGetLocalVecOffset()`, `DMGetGlobalVector()`, `DMNetworkGetComponent()`, `DMCreateGlobalVector()`, `VecGetArray()`, `VecSetValues()`, `MatSetValues()`
    1265: @*/
    1266: PetscErrorCode DMNetworkGetGlobalVecOffset(DM dm, PetscInt p, PetscInt compnum, PetscInt *offsetg)
    1267: {
    1268:   DM_Network              *network = (DM_Network *)dm->data;
    1269:   PetscInt                 offsetp, offsetd;
    1270:   DMNetworkComponentHeader header;

    1272:   PetscFunctionBegin;
    1273:   PetscCall(PetscSectionGetOffset(network->plex->globalSection, p, &offsetp));
    1274:   if (offsetp < 0) offsetp = -(offsetp + 1); /* Convert to actual global offset for ghost vertex */

    1276:   if (compnum == ALL_COMPONENTS) {
    1277:     *offsetg = offsetp;
    1278:     PetscFunctionReturn(PETSC_SUCCESS);
    1279:   }
    1280:   PetscCall(PetscSectionGetOffset(network->DataSection, p, &offsetd));
    1281:   header   = (DMNetworkComponentHeader)(network->componentdataarray + offsetd);
    1282:   *offsetg = offsetp + header->offsetvarrel[compnum];
    1283:   PetscFunctionReturn(PETSC_SUCCESS);
    1284: }

    1286: /*@
    1287:   DMNetworkGetEdgeOffset - Get the offset for accessing the variables associated with the given edge from the local subvector

    1289:   Not Collective

    1291:   Input Parameters:
    1292: + dm - the `DMNETWORK` object
    1293: - p  - the edge point

    1295:   Output Parameter:
    1296: . offset - the offset

    1298:   Level: intermediate

    1300: .seealso: `DM`, `DMNETWORK`, `DMNetworkGetLocalVecOffset()`, `DMGetLocalVector()`
    1301: @*/
    1302: PetscErrorCode DMNetworkGetEdgeOffset(DM dm, PetscInt p, PetscInt *offset)
    1303: {
    1304:   DM_Network *network = (DM_Network *)dm->data;

    1306:   PetscFunctionBegin;
    1307:   PetscCall(PetscSectionGetOffset(network->edge.DofSection, p, offset));
    1308:   PetscFunctionReturn(PETSC_SUCCESS);
    1309: }

    1311: /*@
    1312:   DMNetworkGetVertexOffset - Get the offset for accessing the variables associated with the given vertex from the local subvector

    1314:   Not Collective

    1316:   Input Parameters:
    1317: + dm - the `DMNETWORK` object
    1318: - p  - the vertex point

    1320:   Output Parameter:
    1321: . offset - the offset

    1323:   Level: intermediate

    1325: .seealso: `DM`, `DMNETWORK`, `DMNetworkGetEdgeOffset()`, `DMGetLocalVector()`
    1326: @*/
    1327: PetscErrorCode DMNetworkGetVertexOffset(DM dm, PetscInt p, PetscInt *offset)
    1328: {
    1329:   DM_Network *network = (DM_Network *)dm->data;

    1331:   PetscFunctionBegin;
    1332:   p -= network->cloneshared->vStart;
    1333:   PetscCall(PetscSectionGetOffset(network->vertex.DofSection, p, offset));
    1334:   PetscFunctionReturn(PETSC_SUCCESS);
    1335: }

    1337: /*@
    1338:   DMNetworkAddComponent - Adds a network component and number of variables at the given point (vertex/edge)

    1340:   Collective

    1342:   Input Parameters:
    1343: + dm           - the DMNetwork
    1344: . p            - the vertex/edge point. These points are local indices provided by `DMNetworkGetSubnetwork()`
    1345: . componentkey - component key returned while registering the component with `DMNetworkRegisterComponent()`
    1346: . compvalue    - pointer to the data structure for the component, or `NULL` if the component does not require data, this data is not copied so you cannot
    1347:               free this space until after `DMSetUp()` is called.
    1348: - nvar         - number of variables for the component at the vertex/edge point, zero if the component does not introduce any degrees of freedom at the point

    1350:   Level: beginner

    1352:   Notes:
    1353:   The owning rank and any other ranks that have this point as a ghost location must call this routine to add a component and number of variables in the same order at the given point.

    1355:   `DMNetworkLayoutSetUp()` must be called before this routine.

    1357:   Developer Notes:
    1358:   The requirement that all the ranks with access to a vertex (as owner or as ghost) add all the components comes from a limitation of the underlying implementation based on `DMPLEX`.

    1360: .seealso: `DM`, `DMNETWORK`, `DMNetworkGetComponent()`, `DMNetworkGetSubnetwork()`, `DMNetworkIsGhostVertex()`, `DMNetworkLayoutSetUp()`
    1361: @*/
    1362: PetscErrorCode DMNetworkAddComponent(DM dm, PetscInt p, PetscInt componentkey, void *compvalue, PetscInt nvar)
    1363: {
    1364:   DM_Network              *network   = (DM_Network *)dm->data;
    1365:   DMNetworkComponent      *component = &network->component[componentkey];
    1366:   DMNetworkComponentHeader header;
    1367:   DMNetworkComponentValue  cvalue;
    1368:   PetscInt                 compnum;
    1369:   PetscInt                *compsize, *compkey, *compoffset, *compnvar, *compoffsetvarrel;
    1370:   void                   **compdata;

    1372:   PetscFunctionBegin;
    1373:   PetscCheck(componentkey >= 0, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "componentkey %" PetscInt_FMT " cannot be negative. Input a component key returned while registering the component with DMNetworkRegisterComponent()", componentkey);
    1374:   PetscCheck(network->componentsetup == PETSC_FALSE, PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONGSTATE, "The network has already finalized the components. No new components can be added.");
    1375:   /* The owning rank and all ghost ranks add nvar */
    1376:   PetscCall(PetscSectionAddDof(network->DofSection, p, nvar));

    1378:   /* The owning rank and all ghost ranks add a component, including compvalue=NULL */
    1379:   header = &network->header[p];
    1380:   cvalue = &network->cvalue[p];
    1381:   if (header->ndata == header->maxcomps) {
    1382:     PetscInt additional_size;

    1384:     /* Reached limit so resize header component arrays */
    1385:     header->maxcomps += 2;

    1387:     /* Allocate arrays for component information and value */
    1388:     PetscCall(PetscCalloc5(header->maxcomps, &compsize, header->maxcomps, &compkey, header->maxcomps, &compoffset, header->maxcomps, &compnvar, header->maxcomps, &compoffsetvarrel));
    1389:     PetscCall(PetscMalloc1(header->maxcomps, &compdata));

    1391:     /* Recalculate header size */
    1392:     header->hsize = sizeof(struct _p_DMNetworkComponentHeader) + 5 * header->maxcomps * sizeof(PetscInt);
    1393:     header->hsize /= sizeof(DMNetworkComponentGenericDataType);
    1394: #if defined(__NEC__)
    1395:     /* NEC/LG: quick hack to keep data aligned on 8 bytes. */
    1396:     header->hsize = (header->hsize + (8 - 1)) & ~(8 - 1);
    1397: #endif

    1399:     /* Copy over component info */
    1400:     PetscCall(PetscMemcpy(compsize, header->size, header->ndata * sizeof(PetscInt)));
    1401:     PetscCall(PetscMemcpy(compkey, header->key, header->ndata * sizeof(PetscInt)));
    1402:     PetscCall(PetscMemcpy(compoffset, header->offset, header->ndata * sizeof(PetscInt)));
    1403:     PetscCall(PetscMemcpy(compnvar, header->nvar, header->ndata * sizeof(PetscInt)));
    1404:     PetscCall(PetscMemcpy(compoffsetvarrel, header->offsetvarrel, header->ndata * sizeof(PetscInt)));

    1406:     /* Copy over component data pointers */
    1407:     PetscCall(PetscMemcpy(compdata, cvalue->data, header->ndata * sizeof(void *)));

    1409:     /* Free old arrays */
    1410:     PetscCall(PetscFree5(header->size, header->key, header->offset, header->nvar, header->offsetvarrel));
    1411:     PetscCall(PetscFree(cvalue->data));

    1413:     /* Update pointers */
    1414:     header->size         = compsize;
    1415:     header->key          = compkey;
    1416:     header->offset       = compoffset;
    1417:     header->nvar         = compnvar;
    1418:     header->offsetvarrel = compoffsetvarrel;

    1420:     cvalue->data = compdata;

    1422:     /* Update DataSection Dofs */
    1423:     /* The dofs for datasection point p equals sizeof the header (i.e. header->hsize) + sizes of the components added at point p. With the resizing of the header, we need to update the dofs for point p. Hence, we add the extra size added for the header */
    1424:     additional_size = (5 * (header->maxcomps - header->ndata) * sizeof(PetscInt)) / sizeof(DMNetworkComponentGenericDataType);
    1425:     PetscCall(PetscSectionAddDof(network->DataSection, p, additional_size));
    1426:   }
    1427:   header = &network->header[p];
    1428:   cvalue = &network->cvalue[p];

    1430:   compnum = header->ndata;

    1432:   header->size[compnum] = component->size;
    1433:   PetscCall(PetscSectionAddDof(network->DataSection, p, component->size));
    1434:   header->key[compnum] = componentkey;
    1435:   if (compnum != 0) header->offset[compnum] = header->offset[compnum - 1] + header->size[compnum - 1];
    1436:   cvalue->data[compnum] = compvalue;

    1438:   /* variables */
    1439:   header->nvar[compnum] += nvar;
    1440:   if (compnum != 0) header->offsetvarrel[compnum] = header->offsetvarrel[compnum - 1] + header->nvar[compnum - 1];

    1442:   header->ndata++;
    1443:   PetscFunctionReturn(PETSC_SUCCESS);
    1444: }

    1446: /*@
    1447:   DMNetworkGetComponent - Gets the component key, the component data, and the number of variables at a given network point

    1449:   Not Collective

    1451:   Input Parameters:
    1452: + dm      - the `DMNETWORK` object
    1453: . p       - vertex/edge point
    1454: - compnum - component number; use ALL_COMPONENTS if sum up all the components

    1456:   Output Parameters:
    1457: + compkey   - the key obtained when registering the component (use `NULL` if not required)
    1458: . component - the component data (use `NULL` if not required)
    1459: - nvar      - number of variables (use `NULL` if not required)

    1461:   Level: beginner

    1463: .seealso: `DM`, `DMNETWORK`, `DMNetworkAddComponent()`, `DMNetworkGetNumComponents()`
    1464: @*/
    1465: PetscErrorCode DMNetworkGetComponent(DM dm, PetscInt p, PetscInt compnum, PeOp PetscInt *compkey, PeCtx component, PeOp PetscInt *nvar)
    1466: {
    1467:   DM_Network              *network = (DM_Network *)dm->data;
    1468:   PetscInt                 offset  = 0;
    1469:   DMNetworkComponentHeader header;

    1471:   PetscFunctionBegin;
    1472:   if (compnum == ALL_COMPONENTS) {
    1473:     PetscCall(PetscSectionGetDof(network->DofSection, p, nvar));
    1474:     PetscFunctionReturn(PETSC_SUCCESS);
    1475:   }

    1477:   PetscCall(PetscSectionGetOffset(network->DataSection, p, &offset));
    1478:   header = (DMNetworkComponentHeader)(network->componentdataarray + offset);

    1480:   if (compnum >= 0) {
    1481:     if (compkey) *compkey = header->key[compnum];
    1482:     if (component) {
    1483:       offset += header->hsize + header->offset[compnum];
    1484:       *(void **)component = network->componentdataarray + offset;
    1485:     }
    1486:   }

    1488:   if (nvar) *nvar = header->nvar[compnum];
    1489:   PetscFunctionReturn(PETSC_SUCCESS);
    1490: }

    1492: /*
    1493:  Sets up the array that holds the data for all components and its associated section.
    1494:  It copies the data for all components in a contiguous array called componentdataarray. The component data is stored pointwise with an additional header (metadata) stored for each point. The header has metadata information such as number of components at each point, number of variables for each component, offsets for the components data, etc.
    1495: */
    1496: static PetscErrorCode DMNetworkComponentSetUp(DM dm)
    1497: {
    1498:   DM_Network                        *network = (DM_Network *)dm->data;
    1499:   PetscInt                           arr_size, p, offset, offsetp, ncomp, i, *headerarr;
    1500:   DMNetworkComponentHeader           header;
    1501:   DMNetworkComponentValue            cvalue;
    1502:   DMNetworkComponentHeader           headerinfo;
    1503:   DMNetworkComponentGenericDataType *componentdataarray;

    1505:   PetscFunctionBegin;
    1506:   PetscCall(PetscSectionSetUp(network->DataSection));
    1507:   PetscCall(PetscSectionGetStorageSize(network->DataSection, &arr_size));
    1508:   /* arr_size+1 fixes pipeline test of opensolaris-misc for src/dm/tests/ex10.c -- Do not know why */
    1509:   PetscCall(PetscCalloc1(arr_size + 1, &network->componentdataarray));
    1510:   componentdataarray = network->componentdataarray;
    1511:   for (p = network->cloneshared->pStart; p < network->cloneshared->pEnd; p++) {
    1512:     PetscCall(PetscSectionGetOffset(network->DataSection, p, &offsetp));
    1513:     /* Copy header */
    1514:     header     = &network->header[p];
    1515:     headerinfo = (DMNetworkComponentHeader)(componentdataarray + offsetp);
    1516:     PetscCall(PetscMemcpy(headerinfo, header, sizeof(struct _p_DMNetworkComponentHeader)));
    1517:     headerarr = (PetscInt *)(headerinfo + 1);
    1518:     PetscCall(PetscMemcpy(headerarr, header->size, header->maxcomps * sizeof(PetscInt)));
    1519:     headerinfo->size = headerarr;
    1520:     headerarr += header->maxcomps;
    1521:     PetscCall(PetscMemcpy(headerarr, header->key, header->maxcomps * sizeof(PetscInt)));
    1522:     headerinfo->key = headerarr;
    1523:     headerarr += header->maxcomps;
    1524:     PetscCall(PetscMemcpy(headerarr, header->offset, header->maxcomps * sizeof(PetscInt)));
    1525:     headerinfo->offset = headerarr;
    1526:     headerarr += header->maxcomps;
    1527:     PetscCall(PetscMemcpy(headerarr, header->nvar, header->maxcomps * sizeof(PetscInt)));
    1528:     headerinfo->nvar = headerarr;
    1529:     headerarr += header->maxcomps;
    1530:     PetscCall(PetscMemcpy(headerarr, header->offsetvarrel, header->maxcomps * sizeof(PetscInt)));
    1531:     headerinfo->offsetvarrel = headerarr;

    1533:     /* Copy data */
    1534:     cvalue = &network->cvalue[p];
    1535:     ncomp  = header->ndata;

    1537:     for (i = 0; i < ncomp; i++) {
    1538:       offset = offsetp + header->hsize + header->offset[i];
    1539:       PetscCall(PetscMemcpy(componentdataarray + offset, cvalue->data[i], header->size[i] * sizeof(DMNetworkComponentGenericDataType)));
    1540:     }
    1541:   }

    1543:   for (i = network->cloneshared->pStart; i < network->cloneshared->pEnd; i++) {
    1544:     PetscCall(PetscFree5(network->header[i].size, network->header[i].key, network->header[i].offset, network->header[i].nvar, network->header[i].offsetvarrel));
    1545:     PetscCall(PetscFree(network->cvalue[i].data));
    1546:   }
    1547:   PetscCall(PetscFree2(network->header, network->cvalue));
    1548:   PetscFunctionReturn(PETSC_SUCCESS);
    1549: }

    1551: /* Sets up the section for dofs. This routine is called during DMSetUp() */
    1552: static PetscErrorCode DMNetworkVariablesSetUp(DM dm)
    1553: {
    1554:   DM_Network *network = (DM_Network *)dm->data;

    1556:   PetscFunctionBegin;
    1557:   PetscCall(PetscSectionSetUp(network->DofSection));
    1558:   PetscFunctionReturn(PETSC_SUCCESS);
    1559: }

    1561: /* Get a subsection from a range of points */
    1562: static PetscErrorCode DMNetworkGetSubSection_private(PetscSection main, PetscInt pstart, PetscInt pend, PetscSection *subsection)
    1563: {
    1564:   PetscInt i, nvar;

    1566:   PetscFunctionBegin;
    1567:   PetscCall(PetscSectionCreate(PetscObjectComm((PetscObject)main), subsection));
    1568:   PetscCall(PetscSectionSetChart(*subsection, 0, pend - pstart));
    1569:   for (i = pstart; i < pend; i++) {
    1570:     PetscCall(PetscSectionGetDof(main, i, &nvar));
    1571:     PetscCall(PetscSectionSetDof(*subsection, i - pstart, nvar));
    1572:   }

    1574:   PetscCall(PetscSectionSetUp(*subsection));
    1575:   PetscFunctionReturn(PETSC_SUCCESS);
    1576: }

    1578: /* Create a submap of points with a GlobalToLocal structure */
    1579: static PetscErrorCode DMNetworkSetSubMap_private(DM dm, PetscInt pstart, PetscInt pend, ISLocalToGlobalMapping *map)
    1580: {
    1581:   PetscInt i, *subpoints;

    1583:   PetscFunctionBegin;
    1584:   /* Create index sets to map from "points" to "subpoints" */
    1585:   PetscCall(PetscMalloc1(pend - pstart, &subpoints));
    1586:   for (i = pstart; i < pend; i++) subpoints[i - pstart] = i;
    1587:   PetscCall(ISLocalToGlobalMappingCreate(PetscObjectComm((PetscObject)dm), 1, pend - pstart, subpoints, PETSC_COPY_VALUES, map));
    1588:   PetscCall(PetscFree(subpoints));
    1589:   PetscFunctionReturn(PETSC_SUCCESS);
    1590: }

    1592: /*@
    1593:   DMNetworkAssembleGraphStructures - Assembles vertex and edge data structures. Must be called after `DMNetworkDistribute()`

    1595:   Collective

    1597:   Input Parameter:
    1598: . dm - the `DMNETWORK` Object

    1600:   Level: intermediate

    1602:   Note:
    1603:   The routine will create alternative orderings for the vertices and edges. Assume global
    1604:   network points are\:

    1606:   points = [0 1 2 3 4 5 6]

    1608:   where edges = [0,1,2,3] and vertices = [4,5,6]. The new orderings will be specific to the subset (i.e vertices = [0,1,2] <- [4,5,6]).

    1610:   With this new ordering a local `PetscSection`, global `PetscSection` and` PetscSF` will be created specific to the subset.

    1612: .seealso: `DMNetworkDistribute()`
    1613: @*/
    1614: PetscErrorCode DMNetworkAssembleGraphStructures(DM dm)
    1615: {
    1616:   MPI_Comm    comm;
    1617:   PetscMPIInt size;
    1618:   DM_Network *network = (DM_Network *)dm->data;

    1620:   PetscFunctionBegin;
    1621:   PetscCall(PetscObjectGetComm((PetscObject)dm, &comm));
    1622:   PetscCallMPI(MPI_Comm_size(comm, &size));

    1624:   /* Create maps for vertices and edges */
    1625:   PetscCall(DMNetworkSetSubMap_private(dm, network->cloneshared->vStart, network->cloneshared->vEnd, &network->vertex.mapping));
    1626:   PetscCall(DMNetworkSetSubMap_private(dm, network->cloneshared->eStart, network->cloneshared->eEnd, &network->edge.mapping));

    1628:   /* Create local sub-sections */
    1629:   PetscCall(DMNetworkGetSubSection_private(network->DofSection, network->cloneshared->vStart, network->cloneshared->vEnd, &network->vertex.DofSection));
    1630:   PetscCall(DMNetworkGetSubSection_private(network->DofSection, network->cloneshared->eStart, network->cloneshared->eEnd, &network->edge.DofSection));

    1632:   if (size > 1) {
    1633:     PetscCall(PetscSFGetSubSF(network->plex->sf, network->vertex.mapping, &network->vertex.sf));

    1635:     PetscCall(PetscSectionCreateGlobalSection(network->vertex.DofSection, network->vertex.sf, PETSC_TRUE, PETSC_FALSE, PETSC_FALSE, &network->vertex.GlobalDofSection));
    1636:     PetscCall(PetscSFGetSubSF(network->plex->sf, network->edge.mapping, &network->edge.sf));
    1637:     PetscCall(PetscSectionCreateGlobalSection(network->edge.DofSection, network->edge.sf, PETSC_TRUE, PETSC_FALSE, PETSC_FALSE, &network->edge.GlobalDofSection));
    1638:   } else {
    1639:     /* create structures for vertex */
    1640:     PetscCall(PetscSectionClone(network->vertex.DofSection, &network->vertex.GlobalDofSection));
    1641:     /* create structures for edge */
    1642:     PetscCall(PetscSectionClone(network->edge.DofSection, &network->edge.GlobalDofSection));
    1643:   }

    1645:   /* Add viewers */
    1646:   PetscCall(PetscObjectSetName((PetscObject)network->edge.GlobalDofSection, "Global edge dof section"));
    1647:   PetscCall(PetscObjectSetName((PetscObject)network->vertex.GlobalDofSection, "Global vertex dof section"));
    1648:   PetscCall(PetscSectionViewFromOptions(network->edge.GlobalDofSection, NULL, "-edge_global_section_view"));
    1649:   PetscCall(PetscSectionViewFromOptions(network->vertex.GlobalDofSection, NULL, "-vertex_global_section_view"));
    1650:   PetscFunctionReturn(PETSC_SUCCESS);
    1651: }

    1653: /*
    1654:    Setup a lookup btable for the input v's owning subnetworks
    1655:    - add all owing subnetworks that connect to this v to the btable
    1656:      vertex_subnetid = supportingedge_subnetid
    1657: */
    1658: static inline PetscErrorCode SetSubnetIdLookupBT(DM dm, PetscInt v, PetscInt Nsubnet, PetscBT btable)
    1659: {
    1660:   PetscInt                 e, nedges, offset;
    1661:   const PetscInt          *edges;
    1662:   DM_Network              *newDMnetwork = (DM_Network *)dm->data;
    1663:   DMNetworkComponentHeader header;

    1665:   PetscFunctionBegin;
    1666:   PetscCall(PetscBTMemzero(Nsubnet, btable));
    1667:   PetscCall(DMNetworkGetSupportingEdges(dm, v, &nedges, &edges));
    1668:   for (e = 0; e < nedges; e++) {
    1669:     PetscCall(PetscSectionGetOffset(newDMnetwork->DataSection, edges[e], &offset));
    1670:     header = (DMNetworkComponentHeader)(newDMnetwork->componentdataarray + offset);
    1671:     PetscCall(PetscBTSet(btable, header->subnetid));
    1672:   }
    1673:   PetscFunctionReturn(PETSC_SUCCESS);
    1674: }

    1676: /*
    1677:   DMNetworkDistributeCoordinates - Internal function to distribute the coordinate network and coordinates.

    1679:   Collective

    1681:   Input Parameters:
    1682:   + dm - The original `DMNETWORK` object
    1683:   - migrationSF - The `PetscSF` describing the migration from dm to dmnew
    1684:   - newDM - The new distributed dmnetwork object.
    1685: */

    1687: static PetscErrorCode DMNetworkDistributeCoordinates(DM dm, PetscSF migrationSF, DM newDM)
    1688: {
    1689:   DM_Network              *newDMnetwork = (DM_Network *)((newDM)->data), *newCoordnetwork, *oldCoordnetwork;
    1690:   DM                       cdm, newcdm;
    1691:   PetscInt                 cdim, bs, p, pStart, pEnd, offset;
    1692:   Vec                      oldCoord, newCoord;
    1693:   DMNetworkComponentHeader header;
    1694:   const char              *name;

    1696:   PetscFunctionBegin;
    1697:   /* Distribute the coordinate network and coordinates */
    1698:   PetscCall(DMGetCoordinateDim(dm, &cdim));
    1699:   PetscCall(DMSetCoordinateDim(newDM, cdim));

    1701:   /* Migrate only if original network had coordinates */
    1702:   PetscCall(DMGetCoordinatesLocal(dm, &oldCoord));
    1703:   if (oldCoord) {
    1704:     PetscCall(DMGetCoordinateDM(dm, &cdm));
    1705:     PetscCall(DMGetCoordinateDM(newDM, &newcdm));
    1706:     newCoordnetwork = (DM_Network *)newcdm->data;
    1707:     oldCoordnetwork = (DM_Network *)cdm->data;

    1709:     PetscCall(VecCreate(PETSC_COMM_SELF, &newCoord));
    1710:     PetscCall(PetscObjectGetName((PetscObject)oldCoord, &name));
    1711:     PetscCall(PetscObjectSetName((PetscObject)newCoord, name));
    1712:     PetscCall(VecGetBlockSize(oldCoord, &bs));
    1713:     PetscCall(VecSetBlockSize(newCoord, bs));

    1715:     PetscCall(DMPlexDistributeField(newDMnetwork->plex, migrationSF, oldCoordnetwork->DofSection, oldCoord, newCoordnetwork->DofSection, newCoord));
    1716:     PetscCall(DMSetCoordinatesLocal(newDM, newCoord));

    1718:     PetscCall(VecDestroy(&newCoord));
    1719:     /* Migrate the components from the original coordinate network to the new coordinate network */
    1720:     PetscCall(DMPlexDistributeData(newDMnetwork->plex, migrationSF, oldCoordnetwork->DataSection, MPIU_INT, (void *)oldCoordnetwork->componentdataarray, newCoordnetwork->DataSection, (void **)&newCoordnetwork->componentdataarray));
    1721:     /* update the header pointers in the new coordinate network components */
    1722:     PetscCall(PetscSectionGetChart(newCoordnetwork->DataSection, &pStart, &pEnd));
    1723:     for (p = pStart; p < pEnd; p++) {
    1724:       PetscCall(PetscSectionGetOffset(newCoordnetwork->DataSection, p, &offset));
    1725:       header = (DMNetworkComponentHeader)(newCoordnetwork->componentdataarray + offset);
    1726:       /* Update pointers */
    1727:       header->size         = (PetscInt *)(header + 1);
    1728:       header->key          = header->size + header->maxcomps;
    1729:       header->offset       = header->key + header->maxcomps;
    1730:       header->nvar         = header->offset + header->maxcomps;
    1731:       header->offsetvarrel = header->nvar + header->maxcomps;
    1732:     }

    1734:     PetscCall(DMSetLocalSection(newCoordnetwork->plex, newCoordnetwork->DofSection));
    1735:     PetscCall(DMGetGlobalSection(newCoordnetwork->plex, &newCoordnetwork->GlobalDofSection));
    1736:     newCoordnetwork->componentsetup = PETSC_TRUE;
    1737:   }
    1738:   PetscFunctionReturn(PETSC_SUCCESS);
    1739: }

    1741: /*@
    1742:   DMNetworkDistribute - Distributes the network and moves associated component data

    1744:   Collective

    1746:   Input Parameters:
    1747: + dm      - the `DMNETWORK` object
    1748: - overlap - the overlap of partitions, 0 is the default

    1750:   Options Database Keys:
    1751: + -dmnetwork_view              - Calls `DMView()` at the conclusion of `DMSetUp()`
    1752: . -dmnetwork_view_distributed  - Calls `DMView()` at the conclusion of `DMNetworkDistribute()`
    1753: . -dmnetwork_view_tmpdir       - Sets the temporary directory to use when viewing with the `draw` option
    1754: . -dmnetwork_view_all_ranks    - Displays all of the subnetworks for each MPI rank
    1755: . -dmnetwork_view_rank_range   - Displays the subnetworks for the ranks in a comma-separated list
    1756: . -dmnetwork_view_no_vertices  - Disables displaying the vertices in the network visualization
    1757: - -dmnetwork_view_no_numbering - Disables displaying the numbering of edges and vertices in the network visualization

    1759:   Level: intermediate

    1761:   Note:
    1762:   Distributes the network with <overlap>-overlapping partitioning of the edges.

    1764: .seealso: `DM`, `DMNETWORK`, `DMNetworkCreate()`
    1765: @*/
    1766: PetscErrorCode DMNetworkDistribute(DM *dm, PetscInt overlap)
    1767: {
    1768:   MPI_Comm                 comm;
    1769:   PetscMPIInt              size;
    1770:   DM_Network              *oldDMnetwork = (DM_Network *)((*dm)->data), *newDMnetwork;
    1771:   PetscSF                  pointsf      = NULL;
    1772:   DM                       newDM;
    1773:   PetscInt                 j, e, v, offset, *subnetvtx, *subnetedge, Nsubnet, gidx, svtx_idx, nv, net;
    1774:   PetscInt                *sv;
    1775:   PetscBT                  btable;
    1776:   PetscPartitioner         part;
    1777:   DMNetworkComponentHeader header;

    1779:   PetscFunctionBegin;
    1780:   PetscAssertPointer(dm, 1);
    1782:   PetscCall(PetscObjectGetComm((PetscObject)*dm, &comm));
    1783:   PetscCallMPI(MPI_Comm_size(comm, &size));
    1784:   if (size == 1) {
    1785:     oldDMnetwork->cloneshared->distributecalled = PETSC_TRUE;
    1786:     PetscFunctionReturn(PETSC_SUCCESS);
    1787:   }

    1789:   PetscCheck(!overlap, PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "overlap %" PetscInt_FMT " != 0 is not supported yet", overlap);

    1791:   /* This routine moves the component data to the appropriate processors. It makes use of the DataSection and the componentdataarray to move the component data to appropriate processors and returns a new DataSection and new componentdataarray. */
    1792:   PetscCall(DMNetworkCreate(PetscObjectComm((PetscObject)*dm), &newDM));
    1793:   PetscCall(PetscLogEventBegin(DMNetwork_Distribute, newDM, 0, 0, 0));
    1794:   newDMnetwork                       = (DM_Network *)newDM->data;
    1795:   newDMnetwork->max_comps_registered = oldDMnetwork->max_comps_registered;
    1796:   PetscCall(PetscMalloc1(newDMnetwork->max_comps_registered, &newDMnetwork->component));

    1798:   /* Enable runtime options for petscpartitioner */
    1799:   PetscCall(DMPlexGetPartitioner(oldDMnetwork->plex, &part));
    1800:   PetscCall(PetscPartitionerSetFromOptions(part));

    1802:   /* Distribute plex dm */
    1803:   PetscCall(DMPlexDistribute(oldDMnetwork->plex, overlap, &pointsf, &newDMnetwork->plex));

    1805:   /* Distribute dof section */
    1806:   PetscCall(PetscSectionCreate(comm, &newDMnetwork->DofSection));
    1807:   PetscCall(PetscSFDistributeSection(pointsf, oldDMnetwork->DofSection, NULL, newDMnetwork->DofSection));

    1809:   /* Distribute data and associated section */
    1810:   PetscCall(PetscSectionCreate(comm, &newDMnetwork->DataSection));
    1811:   PetscCall(DMPlexDistributeData(newDMnetwork->plex, pointsf, oldDMnetwork->DataSection, MPIU_INT, (void *)oldDMnetwork->componentdataarray, newDMnetwork->DataSection, (void **)&newDMnetwork->componentdataarray));

    1813:   PetscCall(PetscSectionGetChart(newDMnetwork->DataSection, &newDMnetwork->cloneshared->pStart, &newDMnetwork->cloneshared->pEnd));
    1814:   PetscCall(DMPlexGetHeightStratum(newDMnetwork->plex, 0, &newDMnetwork->cloneshared->eStart, &newDMnetwork->cloneshared->eEnd));
    1815:   PetscCall(DMPlexGetHeightStratum(newDMnetwork->plex, 1, &newDMnetwork->cloneshared->vStart, &newDMnetwork->cloneshared->vEnd));
    1816:   newDMnetwork->cloneshared->nEdges    = newDMnetwork->cloneshared->eEnd - newDMnetwork->cloneshared->eStart;
    1817:   newDMnetwork->cloneshared->nVertices = newDMnetwork->cloneshared->vEnd - newDMnetwork->cloneshared->vStart;
    1818:   newDMnetwork->cloneshared->NVertices = oldDMnetwork->cloneshared->NVertices;
    1819:   newDMnetwork->cloneshared->NEdges    = oldDMnetwork->cloneshared->NEdges;
    1820:   newDMnetwork->cloneshared->svtable   = oldDMnetwork->cloneshared->svtable; /* global table! */
    1821:   oldDMnetwork->cloneshared->svtable   = NULL;

    1823:   /* Set Dof section as the section for dm */
    1824:   PetscCall(DMSetLocalSection(newDMnetwork->plex, newDMnetwork->DofSection));
    1825:   PetscCall(DMGetGlobalSection(newDMnetwork->plex, &newDMnetwork->GlobalDofSection));

    1827:   /* Setup subnetwork info in the newDM */
    1828:   newDMnetwork->cloneshared->Nsubnet = oldDMnetwork->cloneshared->Nsubnet;
    1829:   newDMnetwork->cloneshared->Nsvtx   = oldDMnetwork->cloneshared->Nsvtx;
    1830:   oldDMnetwork->cloneshared->Nsvtx   = 0;
    1831:   newDMnetwork->cloneshared->svtx    = oldDMnetwork->cloneshared->svtx; /* global vertices! */
    1832:   oldDMnetwork->cloneshared->svtx    = NULL;
    1833:   PetscCall(PetscCalloc1(newDMnetwork->cloneshared->Nsubnet, &newDMnetwork->cloneshared->subnet));

    1835:   /* Copy over the global number of vertices and edges in each subnetwork.
    1836:      Note: these are calculated in DMNetworkLayoutSetUp()
    1837:   */
    1838:   Nsubnet = newDMnetwork->cloneshared->Nsubnet;
    1839:   for (j = 0; j < Nsubnet; j++) {
    1840:     newDMnetwork->cloneshared->subnet[j].Nvtx  = oldDMnetwork->cloneshared->subnet[j].Nvtx;
    1841:     newDMnetwork->cloneshared->subnet[j].Nedge = oldDMnetwork->cloneshared->subnet[j].Nedge;
    1842:   }

    1844:   /* Count local nedges for subnetworks */
    1845:   for (e = newDMnetwork->cloneshared->eStart; e < newDMnetwork->cloneshared->eEnd; e++) {
    1846:     PetscCall(PetscSectionGetOffset(newDMnetwork->DataSection, e, &offset));
    1847:     header = (DMNetworkComponentHeader)(newDMnetwork->componentdataarray + offset);

    1849:     /* Update pointers */
    1850:     header->size         = (PetscInt *)(header + 1);
    1851:     header->key          = header->size + header->maxcomps;
    1852:     header->offset       = header->key + header->maxcomps;
    1853:     header->nvar         = header->offset + header->maxcomps;
    1854:     header->offsetvarrel = header->nvar + header->maxcomps;

    1856:     newDMnetwork->cloneshared->subnet[header->subnetid].nedge++;
    1857:   }

    1859:   /* Setup a btable to keep track subnetworks owned by this process at a shared vertex */
    1860:   if (newDMnetwork->cloneshared->Nsvtx) PetscCall(PetscBTCreate(Nsubnet, &btable));

    1862:   /* Count local nvtx for subnetworks */
    1863:   for (v = newDMnetwork->cloneshared->vStart; v < newDMnetwork->cloneshared->vEnd; v++) {
    1864:     PetscCall(PetscSectionGetOffset(newDMnetwork->DataSection, v, &offset));
    1865:     header = (DMNetworkComponentHeader)(newDMnetwork->componentdataarray + offset);

    1867:     /* Update pointers */
    1868:     header->size         = (PetscInt *)(header + 1);
    1869:     header->key          = header->size + header->maxcomps;
    1870:     header->offset       = header->key + header->maxcomps;
    1871:     header->nvar         = header->offset + header->maxcomps;
    1872:     header->offsetvarrel = header->nvar + header->maxcomps;

    1874:     /* shared vertices: use gidx=header->index to check if v is a shared vertex */
    1875:     gidx = header->index;
    1876:     PetscCall(PetscHMapIGetWithDefault(newDMnetwork->cloneshared->svtable, gidx + 1, 0, &svtx_idx));
    1877:     svtx_idx--;

    1879:     if (svtx_idx < 0) { /* not a shared vertex */
    1880:       newDMnetwork->cloneshared->subnet[header->subnetid].nvtx++;
    1881:     } else { /* a shared vertex belongs to more than one subnetworks, it is being counted by multiple subnets */
    1882:       /* Setup a lookup btable for this v's owning subnetworks */
    1883:       PetscCall(SetSubnetIdLookupBT(newDM, v, Nsubnet, btable));

    1885:       for (j = 0; j < newDMnetwork->cloneshared->svtx[svtx_idx].n; j++) {
    1886:         sv  = newDMnetwork->cloneshared->svtx[svtx_idx].sv + 2 * j;
    1887:         net = sv[0];
    1888:         if (PetscBTLookup(btable, net)) newDMnetwork->cloneshared->subnet[net].nvtx++; /* sv is on net owned by this process */
    1889:       }
    1890:     }
    1891:   }

    1893:   /* Get total local nvtx for subnetworks */
    1894:   nv = 0;
    1895:   for (j = 0; j < Nsubnet; j++) nv += newDMnetwork->cloneshared->subnet[j].nvtx;
    1896:   nv += newDMnetwork->cloneshared->Nsvtx;

    1898:   /* Now create the vertices and edge arrays for the subnetworks */
    1899:   PetscCall(PetscCalloc2(newDMnetwork->cloneshared->nEdges, &subnetedge, nv, &subnetvtx)); /* Maps local vertex to local subnetwork's vertex */
    1900:   newDMnetwork->cloneshared->subnetedge = subnetedge;
    1901:   newDMnetwork->cloneshared->subnetvtx  = subnetvtx;
    1902:   for (j = 0; j < newDMnetwork->cloneshared->Nsubnet; j++) {
    1903:     newDMnetwork->cloneshared->subnet[j].edges = subnetedge;
    1904:     subnetedge += newDMnetwork->cloneshared->subnet[j].nedge;

    1906:     newDMnetwork->cloneshared->subnet[j].vertices = subnetvtx;
    1907:     subnetvtx += newDMnetwork->cloneshared->subnet[j].nvtx;

    1909:     /* Temporarily setting nvtx and nedge to 0 so we can use them as counters in the below for loop. These get updated when the vertices and edges are added. */
    1910:     newDMnetwork->cloneshared->subnet[j].nvtx = newDMnetwork->cloneshared->subnet[j].nedge = 0;
    1911:   }
    1912:   newDMnetwork->cloneshared->svertices = subnetvtx;

    1914:   /* Set the edges and vertices in each subnetwork */
    1915:   for (e = newDMnetwork->cloneshared->eStart; e < newDMnetwork->cloneshared->eEnd; e++) {
    1916:     PetscCall(PetscSectionGetOffset(newDMnetwork->DataSection, e, &offset));
    1917:     header                                                                                                                 = (DMNetworkComponentHeader)(newDMnetwork->componentdataarray + offset);
    1918:     newDMnetwork->cloneshared->subnet[header->subnetid].edges[newDMnetwork->cloneshared->subnet[header->subnetid].nedge++] = e;
    1919:   }

    1921:   nv = 0;
    1922:   for (v = newDMnetwork->cloneshared->vStart; v < newDMnetwork->cloneshared->vEnd; v++) {
    1923:     PetscCall(PetscSectionGetOffset(newDMnetwork->DataSection, v, &offset));
    1924:     header = (DMNetworkComponentHeader)(newDMnetwork->componentdataarray + offset);

    1926:     /* coupling vertices: use gidx = header->index to check if v is a coupling vertex */
    1927:     PetscCall(PetscHMapIGetWithDefault(newDMnetwork->cloneshared->svtable, header->index + 1, 0, &svtx_idx));
    1928:     svtx_idx--;
    1929:     if (svtx_idx < 0) {
    1930:       newDMnetwork->cloneshared->subnet[header->subnetid].vertices[newDMnetwork->cloneshared->subnet[header->subnetid].nvtx++] = v;
    1931:     } else { /* a shared vertex */
    1932:       newDMnetwork->cloneshared->svertices[nv++] = v;

    1934:       /* Setup a lookup btable for this v's owning subnetworks */
    1935:       PetscCall(SetSubnetIdLookupBT(newDM, v, Nsubnet, btable));

    1937:       for (j = 0; j < newDMnetwork->cloneshared->svtx[svtx_idx].n; j++) {
    1938:         sv  = newDMnetwork->cloneshared->svtx[svtx_idx].sv + 2 * j;
    1939:         net = sv[0];
    1940:         if (PetscBTLookup(btable, net)) newDMnetwork->cloneshared->subnet[net].vertices[newDMnetwork->cloneshared->subnet[net].nvtx++] = v;
    1941:       }
    1942:     }
    1943:   }
    1944:   newDMnetwork->cloneshared->nsvtx = nv; /* num of local shared vertices */

    1946:   PetscCall(DMNetworkDistributeCoordinates(*dm, pointsf, newDM));
    1947:   newDM->setupcalled                          = (*dm)->setupcalled;
    1948:   newDMnetwork->cloneshared->distributecalled = PETSC_TRUE;

    1950:   /* Free spaces */
    1951:   PetscCall(PetscSFDestroy(&pointsf));
    1952:   PetscCall(DMDestroy(dm));
    1953:   if (newDMnetwork->cloneshared->Nsvtx) PetscCall(PetscBTDestroy(&btable));
    1954:   PetscCall(PetscLogEventEnd(DMNetwork_Distribute, newDM, 0, 0, 0));

    1956:   /* View distributed dmnetwork */
    1957:   PetscCall(DMViewFromOptions(newDM, NULL, "-dmnetwork_view_distributed"));

    1959:   *dm = newDM;
    1960:   PetscFunctionReturn(PETSC_SUCCESS);
    1961: }

    1963: /*@
    1964:   PetscSFGetSubSF - Returns an `PetscSF` for a specific subset of points. Leaves are re-numbered to reflect the new ordering

    1966:   Collective

    1968:   Input Parameters:
    1969: + mainsf - `PetscSF` structure
    1970: - map    - a `ISLocalToGlobalMapping` that contains the subset of points

    1972:   Output Parameter:
    1973: . subSF - a subset of the `mainSF` for the desired subset.

    1975:   Level: intermediate

    1977: .seealso: `PetscSF`
    1978: @*/
    1979: PetscErrorCode PetscSFGetSubSF(PetscSF mainsf, ISLocalToGlobalMapping map, PetscSF *subSF)
    1980: {
    1981:   PetscInt           nroots, nleaves, *ilocal_sub;
    1982:   PetscInt           i, *ilocal_map, nroots_sub, nleaves_sub = 0;
    1983:   PetscInt          *local_points, *remote_points;
    1984:   PetscSFNode       *iremote_sub;
    1985:   const PetscInt    *ilocal;
    1986:   const PetscSFNode *iremote;

    1988:   PetscFunctionBegin;
    1989:   PetscCall(PetscSFGetGraph(mainsf, &nroots, &nleaves, &ilocal, &iremote));

    1991:   /* Look for leaves that pertain to the subset of points. Get the local ordering */
    1992:   PetscCall(PetscMalloc1(nleaves, &ilocal_map));
    1993:   PetscCall(ISGlobalToLocalMappingApply(map, IS_GTOLM_MASK, nleaves, ilocal, NULL, ilocal_map));
    1994:   for (i = 0; i < nleaves; i++) {
    1995:     if (ilocal_map[i] != -1) nleaves_sub += 1;
    1996:   }
    1997:   /* Re-number ilocal with subset numbering. Need information from roots */
    1998:   PetscCall(PetscMalloc2(nroots, &local_points, nroots, &remote_points));
    1999:   for (i = 0; i < nroots; i++) local_points[i] = i;
    2000:   PetscCall(ISGlobalToLocalMappingApply(map, IS_GTOLM_MASK, nroots, local_points, NULL, local_points));
    2001:   PetscCall(PetscSFBcastBegin(mainsf, MPIU_INT, local_points, remote_points, MPI_REPLACE));
    2002:   PetscCall(PetscSFBcastEnd(mainsf, MPIU_INT, local_points, remote_points, MPI_REPLACE));
    2003:   /* Fill up graph using local (that is, local to the subset) numbering. */
    2004:   PetscCall(PetscMalloc1(nleaves_sub, &ilocal_sub));
    2005:   PetscCall(PetscMalloc1(nleaves_sub, &iremote_sub));
    2006:   nleaves_sub = 0;
    2007:   for (i = 0; i < nleaves; i++) {
    2008:     if (ilocal_map[i] != -1) {
    2009:       ilocal_sub[nleaves_sub]        = ilocal_map[i];
    2010:       iremote_sub[nleaves_sub].rank  = iremote[i].rank;
    2011:       iremote_sub[nleaves_sub].index = remote_points[ilocal[i]];
    2012:       nleaves_sub += 1;
    2013:     }
    2014:   }
    2015:   PetscCall(PetscFree2(local_points, remote_points));
    2016:   PetscCall(ISLocalToGlobalMappingGetSize(map, &nroots_sub));

    2018:   /* Create new subSF */
    2019:   PetscCall(PetscSFCreate(PetscObjectComm((PetscObject)mainsf), subSF));
    2020:   PetscCall(PetscSFSetFromOptions(*subSF));
    2021:   PetscCall(PetscSFSetGraph(*subSF, nroots_sub, nleaves_sub, ilocal_sub, PETSC_OWN_POINTER, iremote_sub, PETSC_COPY_VALUES));
    2022:   PetscCall(PetscFree(ilocal_map));
    2023:   PetscCall(PetscFree(iremote_sub));
    2024:   PetscFunctionReturn(PETSC_SUCCESS);
    2025: }

    2027: /*@C
    2028:   DMNetworkGetSupportingEdges - Return the supporting edges for this vertex point

    2030:   Not Collective

    2032:   Input Parameters:
    2033: + dm     - the `DMNETWORK` object
    2034: - vertex - the vertex point

    2036:   Output Parameters:
    2037: + nedges - number of edges connected to this vertex point
    2038: - edges  - list of edge points, pass `NULL` if not needed

    2040:   Level: beginner

    2042: .seealso: `DM`, `DMNETWORK`, `DMNetworkCreate()`, `DMNetworkGetConnectedVertices()`
    2043: @*/
    2044: PetscErrorCode DMNetworkGetSupportingEdges(DM dm, PetscInt vertex, PetscInt *nedges, PeOp const PetscInt *edges[])
    2045: {
    2046:   DM_Network *network = (DM_Network *)dm->data;

    2048:   PetscFunctionBegin;
    2049:   PetscCall(DMPlexGetSupportSize(network->plex, vertex, nedges));
    2050:   if (edges) PetscCall(DMPlexGetSupport(network->plex, vertex, edges));
    2051:   PetscFunctionReturn(PETSC_SUCCESS);
    2052: }

    2054: /*@C
    2055:   DMNetworkGetConnectedVertices - Return the connected vertices for this edge point

    2057:   Not Collective

    2059:   Input Parameters:
    2060: + dm   - the `DMNETWORK` object
    2061: - edge - the edge point

    2063:   Output Parameter:
    2064: . vertices - vertices connected to this edge

    2066:   Level: beginner

    2068: .seealso: `DM`, `DMNETWORK`, `DMNetworkCreate()`, `DMNetworkGetSupportingEdges()`
    2069: @*/
    2070: PetscErrorCode DMNetworkGetConnectedVertices(DM dm, PetscInt edge, const PetscInt *vertices[])
    2071: {
    2072:   DM_Network *network = (DM_Network *)dm->data;

    2074:   PetscFunctionBegin;
    2075:   PetscCall(DMPlexGetCone(network->plex, edge, vertices));
    2076:   PetscFunctionReturn(PETSC_SUCCESS);
    2077: }

    2079: /*@
    2080:   DMNetworkIsSharedVertex - Returns `PETSC_TRUE` if the vertex is shared by subnetworks

    2082:   Not Collective

    2084:   Input Parameters:
    2085: + dm - the `DMNETWORK` object
    2086: - p  - the vertex point

    2088:   Output Parameter:
    2089: . flag - `PETSC_TRUE` if the vertex is shared by subnetworks

    2091:   Level: beginner

    2093: .seealso: `DM`, `DMNETWORK`, `DMNetworkAddSharedVertices()`, `DMNetworkIsGhostVertex()`
    2094: @*/
    2095: PetscErrorCode DMNetworkIsSharedVertex(DM dm, PetscInt p, PetscBool *flag)
    2096: {
    2097:   PetscFunctionBegin;
    2099:   PetscAssertPointer(flag, 3);
    2100:   if (dm->setupcalled) { /* DMNetworkGetGlobalVertexIndex() requires DMSetUp() be called */
    2101:     DM_Network *network = (DM_Network *)dm->data;
    2102:     PetscInt    gidx;

    2104:     PetscCall(DMNetworkGetGlobalVertexIndex(dm, p, &gidx));
    2105:     PetscCall(PetscHMapIHas(network->cloneshared->svtable, gidx + 1, flag));
    2106:   } else { /* would be removed? */
    2107:     PetscInt        nv;
    2108:     const PetscInt *vtx;

    2110:     PetscCall(DMNetworkGetSharedVertices(dm, &nv, &vtx));
    2111:     for (PetscInt i = 0; i < nv; i++) {
    2112:       if (p == vtx[i]) {
    2113:         *flag = PETSC_TRUE;
    2114:         PetscFunctionReturn(PETSC_SUCCESS);
    2115:       }
    2116:     }
    2117:     *flag = PETSC_FALSE;
    2118:   }
    2119:   PetscFunctionReturn(PETSC_SUCCESS);
    2120: }

    2122: /*@
    2123:   DMNetworkIsGhostVertex - Returns `PETSC_TRUE` if the vertex is a ghost vertex

    2125:   Not Collective

    2127:   Input Parameters:
    2128: + dm - the `DMNETWORK` object
    2129: - p  - the vertex point

    2131:   Output Parameter:
    2132: . isghost - `PETSC_TRUE` if the vertex is a ghost point

    2134:   Level: beginner

    2136: .seealso: `DM`, `DMNETWORK`, `DMNetworkGetConnectedVertices()`, `DMNetworkGetVertexRange()`, `DMNetworkIsSharedVertex()`
    2137: @*/
    2138: PetscErrorCode DMNetworkIsGhostVertex(DM dm, PetscInt p, PetscBool *isghost)
    2139: {
    2140:   DM_Network  *network = (DM_Network *)dm->data;
    2141:   PetscInt     offsetg;
    2142:   PetscSection sectiong;

    2144:   PetscFunctionBegin;
    2145:   *isghost = PETSC_FALSE;
    2146:   PetscCall(DMGetGlobalSection(network->plex, &sectiong));
    2147:   PetscCall(PetscSectionGetOffset(sectiong, p, &offsetg));
    2148:   if (offsetg < 0) *isghost = PETSC_TRUE;
    2149:   PetscFunctionReturn(PETSC_SUCCESS);
    2150: }

    2152: PetscErrorCode DMSetUp_Network(DM dm)
    2153: {
    2154:   PetscFunctionBegin;
    2155:   PetscCall(PetscLogEventBegin(DMNetwork_SetUpNetwork, dm, 0, 0, 0));
    2156:   PetscCall(DMNetworkFinalizeComponents(dm));
    2157:   /* View dmnetwork */
    2158:   PetscCall(DMViewFromOptions(dm, NULL, "-dmnetwork_view"));
    2159:   PetscCall(PetscLogEventEnd(DMNetwork_SetUpNetwork, dm, 0, 0, 0));
    2160:   PetscFunctionReturn(PETSC_SUCCESS);
    2161: }

    2163: /*@
    2164:   DMNetworkHasJacobian - Sets global flag for using user's sub Jacobian matrices
    2165:   -- replaced by DMNetworkSetOption(network,userjacobian,PETSC_TRUE)?

    2167:   Collective

    2169:   Input Parameters:
    2170: + dm   - the `DMNETWORK` object
    2171: . eflg - turn the option on (`PETSC_TRUE`) or off (`PETSC_FALSE`) if user provides Jacobian for edges
    2172: - vflg - turn the option on (`PETSC_TRUE`) or off (`PETSC_FALSE`) if user provides Jacobian for vertices

    2174:   Level: intermediate

    2176: .seealso: `DMNetworkSetOption()`
    2177: @*/
    2178: PetscErrorCode DMNetworkHasJacobian(DM dm, PetscBool eflg, PetscBool vflg)
    2179: {
    2180:   DM_Network *network   = (DM_Network *)dm->data;
    2181:   PetscInt    nVertices = network->cloneshared->nVertices;

    2183:   PetscFunctionBegin;
    2184:   network->userEdgeJacobian   = eflg;
    2185:   network->userVertexJacobian = vflg;

    2187:   if (eflg && !network->Je) PetscCall(PetscCalloc1(3 * network->cloneshared->nEdges, &network->Je));

    2189:   if (vflg && !network->Jv && nVertices) {
    2190:     PetscInt        i, *vptr, nedges, vStart = network->cloneshared->vStart;
    2191:     PetscInt        nedges_total;
    2192:     const PetscInt *edges;

    2194:     /* count nvertex_total */
    2195:     nedges_total = 0;
    2196:     PetscCall(PetscMalloc1(nVertices + 1, &vptr));

    2198:     vptr[0] = 0;
    2199:     for (i = 0; i < nVertices; i++) {
    2200:       PetscCall(DMNetworkGetSupportingEdges(dm, i + vStart, &nedges, &edges));
    2201:       nedges_total += nedges;
    2202:       vptr[i + 1] = vptr[i] + 2 * nedges + 1;
    2203:     }

    2205:     PetscCall(PetscCalloc1(2 * nedges_total + nVertices, &network->Jv));
    2206:     network->Jvptr = vptr;
    2207:   }
    2208:   PetscFunctionReturn(PETSC_SUCCESS);
    2209: }

    2211: /*@
    2212:   DMNetworkEdgeSetMatrix - Sets user-provided Jacobian matrices for this edge to the network

    2214:   Not Collective

    2216:   Input Parameters:
    2217: + dm - the `DMNETWORK` object
    2218: . p  - the edge point
    2219: - J  - array (size = 3) of Jacobian submatrices for this edge point:
    2220:         J[0]: this edge
    2221:         J[1] and J[2]: connected vertices, obtained by calling `DMNetworkGetConnectedVertices()`

    2223:   Level: advanced

    2225: .seealso: `DM`, `DMNETWORK`, `DMNetworkVertexSetMatrix()`
    2226: @*/
    2227: PetscErrorCode DMNetworkEdgeSetMatrix(DM dm, PetscInt p, Mat J[])
    2228: {
    2229:   DM_Network *network = (DM_Network *)dm->data;

    2231:   PetscFunctionBegin;
    2232:   PetscCheck(network->Je, PetscObjectComm((PetscObject)dm), PETSC_ERR_ORDER, "Must call DMNetworkHasJacobian() collectively before calling DMNetworkEdgeSetMatrix");

    2234:   if (J) {
    2235:     network->Je[3 * p]     = J[0];
    2236:     network->Je[3 * p + 1] = J[1];
    2237:     network->Je[3 * p + 2] = J[2];
    2238:   }
    2239:   PetscFunctionReturn(PETSC_SUCCESS);
    2240: }

    2242: /*@
    2243:   DMNetworkVertexSetMatrix - Sets user-provided Jacobian matrix for this vertex to the network

    2245:   Not Collective

    2247:   Input Parameters:
    2248: + dm - The `DMNETWORK` object
    2249: . p  - the vertex point
    2250: - J  - array of Jacobian (size = 2*(num of supporting edges) + 1) submatrices for this vertex point:
    2251:         J[0]:       this vertex
    2252:         J[1+2*i]:   i-th supporting edge
    2253:         J[1+2*i+1]: i-th connected vertex

    2255:   Level: advanced

    2257: .seealso: `DM`, `DMNETWORK`, `DMNetworkEdgeSetMatrix()`
    2258: @*/
    2259: PetscErrorCode DMNetworkVertexSetMatrix(DM dm, PetscInt p, Mat J[])
    2260: {
    2261:   DM_Network     *network = (DM_Network *)dm->data;
    2262:   PetscInt        i, *vptr, nedges, vStart = network->cloneshared->vStart;
    2263:   const PetscInt *edges;

    2265:   PetscFunctionBegin;
    2266:   PetscCheck(network->Jv, PetscObjectComm((PetscObject)dm), PETSC_ERR_ORDER, "Must call DMNetworkHasJacobian() collectively before calling DMNetworkVertexSetMatrix");

    2268:   if (J) {
    2269:     vptr                          = network->Jvptr;
    2270:     network->Jv[vptr[p - vStart]] = J[0]; /* Set Jacobian for this vertex */

    2272:     /* Set Jacobian for each supporting edge and connected vertex */
    2273:     PetscCall(DMNetworkGetSupportingEdges(dm, p, &nedges, &edges));
    2274:     for (i = 1; i <= 2 * nedges; i++) network->Jv[vptr[p - vStart] + i] = J[i];
    2275:   }
    2276:   PetscFunctionReturn(PETSC_SUCCESS);
    2277: }

    2279: static inline PetscErrorCode MatSetPreallocationDenseblock_private(PetscInt nrows, PetscInt *rows, PetscInt ncols, PetscBool ghost, Vec vdnz, Vec vonz)
    2280: {
    2281:   PetscInt    j;
    2282:   PetscScalar val = (PetscScalar)ncols;

    2284:   PetscFunctionBegin;
    2285:   if (!ghost) {
    2286:     for (j = 0; j < nrows; j++) PetscCall(VecSetValues(vdnz, 1, &rows[j], &val, ADD_VALUES));
    2287:   } else {
    2288:     for (j = 0; j < nrows; j++) PetscCall(VecSetValues(vonz, 1, &rows[j], &val, ADD_VALUES));
    2289:   }
    2290:   PetscFunctionReturn(PETSC_SUCCESS);
    2291: }

    2293: static inline PetscErrorCode MatSetPreallocationUserblock_private(Mat Ju, PetscInt nrows, PetscInt *rows, PetscInt ncols, PetscBool ghost, Vec vdnz, Vec vonz)
    2294: {
    2295:   PetscInt    j, ncols_u;
    2296:   PetscScalar val;

    2298:   PetscFunctionBegin;
    2299:   if (!ghost) {
    2300:     for (j = 0; j < nrows; j++) {
    2301:       PetscCall(MatGetRow(Ju, j, &ncols_u, NULL, NULL));
    2302:       val = (PetscScalar)ncols_u;
    2303:       PetscCall(VecSetValues(vdnz, 1, &rows[j], &val, ADD_VALUES));
    2304:       PetscCall(MatRestoreRow(Ju, j, &ncols_u, NULL, NULL));
    2305:     }
    2306:   } else {
    2307:     for (j = 0; j < nrows; j++) {
    2308:       PetscCall(MatGetRow(Ju, j, &ncols_u, NULL, NULL));
    2309:       val = (PetscScalar)ncols_u;
    2310:       PetscCall(VecSetValues(vonz, 1, &rows[j], &val, ADD_VALUES));
    2311:       PetscCall(MatRestoreRow(Ju, j, &ncols_u, NULL, NULL));
    2312:     }
    2313:   }
    2314:   PetscFunctionReturn(PETSC_SUCCESS);
    2315: }

    2317: static inline PetscErrorCode MatSetPreallocationblock_private(Mat Ju, PetscInt nrows, PetscInt *rows, PetscInt ncols, PetscBool ghost, Vec vdnz, Vec vonz)
    2318: {
    2319:   PetscFunctionBegin;
    2320:   if (Ju) {
    2321:     PetscCall(MatSetPreallocationUserblock_private(Ju, nrows, rows, ncols, ghost, vdnz, vonz));
    2322:   } else {
    2323:     PetscCall(MatSetPreallocationDenseblock_private(nrows, rows, ncols, ghost, vdnz, vonz));
    2324:   }
    2325:   PetscFunctionReturn(PETSC_SUCCESS);
    2326: }

    2328: static inline PetscErrorCode MatSetDenseblock_private(PetscInt nrows, PetscInt *rows, PetscInt ncols, PetscInt cstart, Mat *J)
    2329: {
    2330:   PetscInt     j, *cols;
    2331:   PetscScalar *zeros;

    2333:   PetscFunctionBegin;
    2334:   PetscCall(PetscCalloc2(ncols, &cols, nrows * ncols, &zeros));
    2335:   for (j = 0; j < ncols; j++) cols[j] = j + cstart;
    2336:   PetscCall(MatSetValues(*J, nrows, rows, ncols, cols, zeros, INSERT_VALUES));
    2337:   PetscCall(PetscFree2(cols, zeros));
    2338:   PetscFunctionReturn(PETSC_SUCCESS);
    2339: }

    2341: static inline PetscErrorCode MatSetUserblock_private(Mat Ju, PetscInt nrows, PetscInt *rows, PetscInt ncols, PetscInt cstart, Mat *J)
    2342: {
    2343:   PetscInt        j, M, N, row, col, ncols_u;
    2344:   const PetscInt *cols;
    2345:   PetscScalar     zero = 0.0;

    2347:   PetscFunctionBegin;
    2348:   PetscCall(MatGetSize(Ju, &M, &N));
    2349:   PetscCheck(nrows == M && ncols == N, PetscObjectComm((PetscObject)Ju), PETSC_ERR_USER, "%" PetscInt_FMT " by %" PetscInt_FMT " must equal %" PetscInt_FMT " by %" PetscInt_FMT, nrows, ncols, M, N);

    2351:   for (row = 0; row < nrows; row++) {
    2352:     PetscCall(MatGetRow(Ju, row, &ncols_u, &cols, NULL));
    2353:     for (j = 0; j < ncols_u; j++) {
    2354:       col = cols[j] + cstart;
    2355:       PetscCall(MatSetValues(*J, 1, &rows[row], 1, &col, &zero, INSERT_VALUES));
    2356:     }
    2357:     PetscCall(MatRestoreRow(Ju, row, &ncols_u, &cols, NULL));
    2358:   }
    2359:   PetscFunctionReturn(PETSC_SUCCESS);
    2360: }

    2362: static inline PetscErrorCode MatSetblock_private(Mat Ju, PetscInt nrows, PetscInt *rows, PetscInt ncols, PetscInt cstart, Mat *J)
    2363: {
    2364:   PetscFunctionBegin;
    2365:   if (Ju) {
    2366:     PetscCall(MatSetUserblock_private(Ju, nrows, rows, ncols, cstart, J));
    2367:   } else {
    2368:     PetscCall(MatSetDenseblock_private(nrows, rows, ncols, cstart, J));
    2369:   }
    2370:   PetscFunctionReturn(PETSC_SUCCESS);
    2371: }

    2373: /* Creates a GlobalToLocal mapping with a Local and Global section. This is akin to the routine DMGetLocalToGlobalMapping but without the need of providing a dm.
    2374: */
    2375: static PetscErrorCode CreateSubGlobalToLocalMapping_private(PetscSection globalsec, PetscSection localsec, ISLocalToGlobalMapping *ltog)
    2376: {
    2377:   PetscInt  i, size, dof;
    2378:   PetscInt *glob2loc;

    2380:   PetscFunctionBegin;
    2381:   PetscCall(PetscSectionGetStorageSize(localsec, &size));
    2382:   PetscCall(PetscMalloc1(size, &glob2loc));

    2384:   for (i = 0; i < size; i++) {
    2385:     PetscCall(PetscSectionGetOffset(globalsec, i, &dof));
    2386:     dof         = (dof >= 0) ? dof : -(dof + 1);
    2387:     glob2loc[i] = dof;
    2388:   }

    2390:   PetscCall(ISLocalToGlobalMappingCreate(PetscObjectComm((PetscObject)globalsec), 1, size, glob2loc, PETSC_OWN_POINTER, ltog));
    2391: #if 0
    2392:   PetscCall(PetscIntView(size,glob2loc,PETSC_VIEWER_STDOUT_WORLD));
    2393: #endif
    2394:   PetscFunctionReturn(PETSC_SUCCESS);
    2395: }

    2397: #include <petsc/private/matimpl.h>

    2399: static PetscErrorCode DMCreateMatrix_Network_Nest(DM dm, Mat *J)
    2400: {
    2401:   DM_Network            *network = (DM_Network *)dm->data;
    2402:   PetscInt               eDof, vDof;
    2403:   Mat                    j11, j12, j21, j22, bA[2][2];
    2404:   MPI_Comm               comm;
    2405:   ISLocalToGlobalMapping eISMap, vISMap;

    2407:   PetscFunctionBegin;
    2408:   PetscCall(PetscObjectGetComm((PetscObject)dm, &comm));

    2410:   PetscCall(PetscSectionGetConstrainedStorageSize(network->edge.GlobalDofSection, &eDof));
    2411:   PetscCall(PetscSectionGetConstrainedStorageSize(network->vertex.GlobalDofSection, &vDof));

    2413:   PetscCall(MatCreate(comm, &j11));
    2414:   PetscCall(MatSetSizes(j11, eDof, eDof, PETSC_DETERMINE, PETSC_DETERMINE));
    2415:   PetscCall(MatSetType(j11, MATMPIAIJ));

    2417:   PetscCall(MatCreate(comm, &j12));
    2418:   PetscCall(MatSetSizes(j12, eDof, vDof, PETSC_DETERMINE, PETSC_DETERMINE));
    2419:   PetscCall(MatSetType(j12, MATMPIAIJ));

    2421:   PetscCall(MatCreate(comm, &j21));
    2422:   PetscCall(MatSetSizes(j21, vDof, eDof, PETSC_DETERMINE, PETSC_DETERMINE));
    2423:   PetscCall(MatSetType(j21, MATMPIAIJ));

    2425:   PetscCall(MatCreate(comm, &j22));
    2426:   PetscCall(MatSetSizes(j22, vDof, vDof, PETSC_DETERMINE, PETSC_DETERMINE));
    2427:   PetscCall(MatSetType(j22, MATMPIAIJ));

    2429:   bA[0][0] = j11;
    2430:   bA[0][1] = j12;
    2431:   bA[1][0] = j21;
    2432:   bA[1][1] = j22;

    2434:   PetscCall(CreateSubGlobalToLocalMapping_private(network->edge.GlobalDofSection, network->edge.DofSection, &eISMap));
    2435:   PetscCall(CreateSubGlobalToLocalMapping_private(network->vertex.GlobalDofSection, network->vertex.DofSection, &vISMap));

    2437:   PetscCall(MatSetLocalToGlobalMapping(j11, eISMap, eISMap));
    2438:   PetscCall(MatSetLocalToGlobalMapping(j12, eISMap, vISMap));
    2439:   PetscCall(MatSetLocalToGlobalMapping(j21, vISMap, eISMap));
    2440:   PetscCall(MatSetLocalToGlobalMapping(j22, vISMap, vISMap));

    2442:   PetscCall(MatSetUp(j11));
    2443:   PetscCall(MatSetUp(j12));
    2444:   PetscCall(MatSetUp(j21));
    2445:   PetscCall(MatSetUp(j22));

    2447:   PetscCall(MatCreateNest(comm, 2, NULL, 2, NULL, &bA[0][0], J));
    2448:   PetscCall(MatSetUp(*J));
    2449:   PetscCall(MatNestSetVecType(*J, VECNEST));
    2450:   PetscCall(MatDestroy(&j11));
    2451:   PetscCall(MatDestroy(&j12));
    2452:   PetscCall(MatDestroy(&j21));
    2453:   PetscCall(MatDestroy(&j22));

    2455:   PetscCall(MatAssemblyBegin(*J, MAT_FINAL_ASSEMBLY));
    2456:   PetscCall(MatAssemblyEnd(*J, MAT_FINAL_ASSEMBLY));
    2457:   PetscCall(MatSetOption(*J, MAT_NEW_NONZERO_ALLOCATION_ERR, PETSC_TRUE));

    2459:   /* Free structures */
    2460:   PetscCall(ISLocalToGlobalMappingDestroy(&eISMap));
    2461:   PetscCall(ISLocalToGlobalMappingDestroy(&vISMap));
    2462:   PetscFunctionReturn(PETSC_SUCCESS);
    2463: }

    2465: PetscErrorCode DMCreateMatrix_Network(DM dm, Mat *J)
    2466: {
    2467:   DM_Network     *network = (DM_Network *)dm->data;
    2468:   PetscInt        eStart, eEnd, vStart, vEnd, rstart, nrows, *rows, localSize;
    2469:   PetscInt        cstart, ncols, j, e, v;
    2470:   PetscBool       ghost, ghost_vc, ghost2, isNest;
    2471:   Mat             Juser;
    2472:   PetscSection    sectionGlobal;
    2473:   PetscInt        nedges, *vptr = NULL, vc, *rows_v; /* suppress maybe-uninitialized warning */
    2474:   const PetscInt *edges, *cone;
    2475:   MPI_Comm        comm;
    2476:   MatType         mtype;
    2477:   Vec             vd_nz, vo_nz;
    2478:   PetscInt       *dnnz, *onnz;
    2479:   PetscScalar    *vdnz, *vonz;

    2481:   PetscFunctionBegin;
    2482:   mtype = dm->mattype;
    2483:   PetscCall(PetscStrcmp(mtype, MATNEST, &isNest));
    2484:   if (isNest) {
    2485:     PetscCall(DMCreateMatrix_Network_Nest(dm, J));
    2486:     PetscCall(MatSetDM(*J, dm));
    2487:     PetscFunctionReturn(PETSC_SUCCESS);
    2488:   }

    2490:   if (!network->userEdgeJacobian && !network->userVertexJacobian) {
    2491:     /* user does not provide Jacobian blocks */
    2492:     PetscCall(DMCreateMatrix_Plex(network->plex, J));
    2493:     PetscCall(MatSetDM(*J, dm));
    2494:     PetscFunctionReturn(PETSC_SUCCESS);
    2495:   }

    2497:   PetscCall(MatCreate(PetscObjectComm((PetscObject)dm), J));
    2498:   PetscCall(DMGetGlobalSection(network->plex, &sectionGlobal));
    2499:   PetscCall(PetscSectionGetConstrainedStorageSize(sectionGlobal, &localSize));
    2500:   PetscCall(MatSetSizes(*J, localSize, localSize, PETSC_DETERMINE, PETSC_DETERMINE));

    2502:   PetscCall(MatSetType(*J, MATAIJ));
    2503:   PetscCall(MatSetFromOptions(*J));

    2505:   /* (1) Set matrix preallocation */
    2506:   /*------------------------------*/
    2507:   PetscCall(PetscObjectGetComm((PetscObject)dm, &comm));
    2508:   PetscCall(VecCreate(comm, &vd_nz));
    2509:   PetscCall(VecSetSizes(vd_nz, localSize, PETSC_DECIDE));
    2510:   PetscCall(VecSetFromOptions(vd_nz));
    2511:   PetscCall(VecSet(vd_nz, 0.0));
    2512:   PetscCall(VecDuplicate(vd_nz, &vo_nz));

    2514:   /* Set preallocation for edges */
    2515:   /*-----------------------------*/
    2516:   PetscCall(DMNetworkGetEdgeRange(dm, &eStart, &eEnd));

    2518:   PetscCall(PetscMalloc1(localSize, &rows));
    2519:   for (e = eStart; e < eEnd; e++) {
    2520:     /* Get row indices */
    2521:     PetscCall(DMNetworkGetGlobalVecOffset(dm, e, ALL_COMPONENTS, &rstart));
    2522:     PetscCall(PetscSectionGetDof(network->DofSection, e, &nrows));
    2523:     if (nrows) {
    2524:       for (j = 0; j < nrows; j++) rows[j] = j + rstart;

    2526:       /* Set preallocation for connected vertices */
    2527:       PetscCall(DMNetworkGetConnectedVertices(dm, e, &cone));
    2528:       for (v = 0; v < 2; v++) {
    2529:         PetscCall(PetscSectionGetDof(network->DofSection, cone[v], &ncols));

    2531:         if (network->Je) {
    2532:           Juser = network->Je[3 * e + 1 + v]; /* Jacobian(e,v) */
    2533:         } else Juser = NULL;
    2534:         PetscCall(DMNetworkIsGhostVertex(dm, cone[v], &ghost));
    2535:         PetscCall(MatSetPreallocationblock_private(Juser, nrows, rows, ncols, ghost, vd_nz, vo_nz));
    2536:       }

    2538:       /* Set preallocation for edge self */
    2539:       cstart = rstart;
    2540:       if (network->Je) {
    2541:         Juser = network->Je[3 * e]; /* Jacobian(e,e) */
    2542:       } else Juser = NULL;
    2543:       PetscCall(MatSetPreallocationblock_private(Juser, nrows, rows, nrows, PETSC_FALSE, vd_nz, vo_nz));
    2544:     }
    2545:   }

    2547:   /* Set preallocation for vertices */
    2548:   /*--------------------------------*/
    2549:   PetscCall(DMNetworkGetVertexRange(dm, &vStart, &vEnd));
    2550:   if (vEnd - vStart) vptr = network->Jvptr;

    2552:   for (v = vStart; v < vEnd; v++) {
    2553:     /* Get row indices */
    2554:     PetscCall(DMNetworkGetGlobalVecOffset(dm, v, ALL_COMPONENTS, &rstart));
    2555:     PetscCall(PetscSectionGetDof(network->DofSection, v, &nrows));
    2556:     if (!nrows) continue;

    2558:     PetscCall(DMNetworkIsGhostVertex(dm, v, &ghost));
    2559:     if (ghost) {
    2560:       PetscCall(PetscMalloc1(nrows, &rows_v));
    2561:     } else {
    2562:       rows_v = rows;
    2563:     }

    2565:     for (j = 0; j < nrows; j++) rows_v[j] = j + rstart;

    2567:     /* Get supporting edges and connected vertices */
    2568:     PetscCall(DMNetworkGetSupportingEdges(dm, v, &nedges, &edges));

    2570:     for (e = 0; e < nedges; e++) {
    2571:       /* Supporting edges */
    2572:       PetscCall(DMNetworkGetGlobalVecOffset(dm, edges[e], ALL_COMPONENTS, &cstart));
    2573:       PetscCall(PetscSectionGetDof(network->DofSection, edges[e], &ncols));

    2575:       if (network->Jv) {
    2576:         Juser = network->Jv[vptr[v - vStart] + 2 * e + 1]; /* Jacobian(v,e) */
    2577:       } else Juser = NULL;
    2578:       PetscCall(MatSetPreallocationblock_private(Juser, nrows, rows_v, ncols, ghost, vd_nz, vo_nz));

    2580:       /* Connected vertices */
    2581:       PetscCall(DMNetworkGetConnectedVertices(dm, edges[e], &cone));
    2582:       vc = (v == cone[0]) ? cone[1] : cone[0];
    2583:       PetscCall(DMNetworkIsGhostVertex(dm, vc, &ghost_vc));

    2585:       PetscCall(PetscSectionGetDof(network->DofSection, vc, &ncols));

    2587:       if (network->Jv) {
    2588:         Juser = network->Jv[vptr[v - vStart] + 2 * e + 2]; /* Jacobian(v,vc) */
    2589:       } else Juser = NULL;
    2590:       if (ghost_vc || ghost) {
    2591:         ghost2 = PETSC_TRUE;
    2592:       } else {
    2593:         ghost2 = PETSC_FALSE;
    2594:       }
    2595:       PetscCall(MatSetPreallocationblock_private(Juser, nrows, rows_v, ncols, ghost2, vd_nz, vo_nz));
    2596:     }

    2598:     /* Set preallocation for vertex self */
    2599:     PetscCall(DMNetworkIsGhostVertex(dm, v, &ghost));
    2600:     if (!ghost) {
    2601:       PetscCall(DMNetworkGetGlobalVecOffset(dm, v, ALL_COMPONENTS, &cstart));
    2602:       if (network->Jv) {
    2603:         Juser = network->Jv[vptr[v - vStart]]; /* Jacobian(v,v) */
    2604:       } else Juser = NULL;
    2605:       PetscCall(MatSetPreallocationblock_private(Juser, nrows, rows_v, nrows, PETSC_FALSE, vd_nz, vo_nz));
    2606:     }
    2607:     if (ghost) PetscCall(PetscFree(rows_v));
    2608:   }

    2610:   PetscCall(VecAssemblyBegin(vd_nz));
    2611:   PetscCall(VecAssemblyBegin(vo_nz));

    2613:   PetscCall(PetscMalloc2(localSize, &dnnz, localSize, &onnz));

    2615:   PetscCall(VecAssemblyEnd(vd_nz));
    2616:   PetscCall(VecAssemblyEnd(vo_nz));

    2618:   PetscCall(VecGetArray(vd_nz, &vdnz));
    2619:   PetscCall(VecGetArray(vo_nz, &vonz));
    2620:   for (j = 0; j < localSize; j++) {
    2621:     dnnz[j] = (PetscInt)PetscRealPart(vdnz[j]);
    2622:     onnz[j] = (PetscInt)PetscRealPart(vonz[j]);
    2623:   }
    2624:   PetscCall(VecRestoreArray(vd_nz, &vdnz));
    2625:   PetscCall(VecRestoreArray(vo_nz, &vonz));
    2626:   PetscCall(VecDestroy(&vd_nz));
    2627:   PetscCall(VecDestroy(&vo_nz));

    2629:   PetscCall(MatSeqAIJSetPreallocation(*J, 0, dnnz));
    2630:   PetscCall(MatMPIAIJSetPreallocation(*J, 0, dnnz, 0, onnz));
    2631:   PetscCall(MatSetOption(*J, MAT_NEW_NONZERO_ALLOCATION_ERR, PETSC_TRUE));

    2633:   PetscCall(PetscFree2(dnnz, onnz));

    2635:   /* (2) Set matrix entries for edges */
    2636:   /*----------------------------------*/
    2637:   for (e = eStart; e < eEnd; e++) {
    2638:     /* Get row indices */
    2639:     PetscCall(DMNetworkGetGlobalVecOffset(dm, e, ALL_COMPONENTS, &rstart));
    2640:     PetscCall(PetscSectionGetDof(network->DofSection, e, &nrows));
    2641:     if (nrows) {
    2642:       for (j = 0; j < nrows; j++) rows[j] = j + rstart;

    2644:       /* Set matrix entries for connected vertices */
    2645:       PetscCall(DMNetworkGetConnectedVertices(dm, e, &cone));
    2646:       for (v = 0; v < 2; v++) {
    2647:         PetscCall(DMNetworkGetGlobalVecOffset(dm, cone[v], ALL_COMPONENTS, &cstart));
    2648:         PetscCall(PetscSectionGetDof(network->DofSection, cone[v], &ncols));

    2650:         if (network->Je) {
    2651:           Juser = network->Je[3 * e + 1 + v]; /* Jacobian(e,v) */
    2652:         } else Juser = NULL;
    2653:         PetscCall(MatSetblock_private(Juser, nrows, rows, ncols, cstart, J));
    2654:       }

    2656:       /* Set matrix entries for edge self */
    2657:       cstart = rstart;
    2658:       if (network->Je) {
    2659:         Juser = network->Je[3 * e]; /* Jacobian(e,e) */
    2660:       } else Juser = NULL;
    2661:       PetscCall(MatSetblock_private(Juser, nrows, rows, nrows, cstart, J));
    2662:     }
    2663:   }

    2665:   /* Set matrix entries for vertices */
    2666:   /*---------------------------------*/
    2667:   for (v = vStart; v < vEnd; v++) {
    2668:     /* Get row indices */
    2669:     PetscCall(DMNetworkGetGlobalVecOffset(dm, v, ALL_COMPONENTS, &rstart));
    2670:     PetscCall(PetscSectionGetDof(network->DofSection, v, &nrows));
    2671:     if (!nrows) continue;

    2673:     PetscCall(DMNetworkIsGhostVertex(dm, v, &ghost));
    2674:     if (ghost) {
    2675:       PetscCall(PetscMalloc1(nrows, &rows_v));
    2676:     } else {
    2677:       rows_v = rows;
    2678:     }
    2679:     for (j = 0; j < nrows; j++) rows_v[j] = j + rstart;

    2681:     /* Get supporting edges and connected vertices */
    2682:     PetscCall(DMNetworkGetSupportingEdges(dm, v, &nedges, &edges));

    2684:     for (e = 0; e < nedges; e++) {
    2685:       /* Supporting edges */
    2686:       PetscCall(DMNetworkGetGlobalVecOffset(dm, edges[e], ALL_COMPONENTS, &cstart));
    2687:       PetscCall(PetscSectionGetDof(network->DofSection, edges[e], &ncols));

    2689:       if (network->Jv) {
    2690:         Juser = network->Jv[vptr[v - vStart] + 2 * e + 1]; /* Jacobian(v,e) */
    2691:       } else Juser = NULL;
    2692:       PetscCall(MatSetblock_private(Juser, nrows, rows_v, ncols, cstart, J));

    2694:       /* Connected vertices */
    2695:       PetscCall(DMNetworkGetConnectedVertices(dm, edges[e], &cone));
    2696:       vc = (v == cone[0]) ? cone[1] : cone[0];

    2698:       PetscCall(DMNetworkGetGlobalVecOffset(dm, vc, ALL_COMPONENTS, &cstart));
    2699:       PetscCall(PetscSectionGetDof(network->DofSection, vc, &ncols));

    2701:       if (network->Jv) {
    2702:         Juser = network->Jv[vptr[v - vStart] + 2 * e + 2]; /* Jacobian(v,vc) */
    2703:       } else Juser = NULL;
    2704:       PetscCall(MatSetblock_private(Juser, nrows, rows_v, ncols, cstart, J));
    2705:     }

    2707:     /* Set matrix entries for vertex self */
    2708:     if (!ghost) {
    2709:       PetscCall(DMNetworkGetGlobalVecOffset(dm, v, ALL_COMPONENTS, &cstart));
    2710:       if (network->Jv) {
    2711:         Juser = network->Jv[vptr[v - vStart]]; /* Jacobian(v,v) */
    2712:       } else Juser = NULL;
    2713:       PetscCall(MatSetblock_private(Juser, nrows, rows_v, nrows, cstart, J));
    2714:     }
    2715:     if (ghost) PetscCall(PetscFree(rows_v));
    2716:   }
    2717:   PetscCall(PetscFree(rows));

    2719:   PetscCall(MatAssemblyBegin(*J, MAT_FINAL_ASSEMBLY));
    2720:   PetscCall(MatAssemblyEnd(*J, MAT_FINAL_ASSEMBLY));

    2722:   PetscCall(MatSetDM(*J, dm));
    2723:   PetscFunctionReturn(PETSC_SUCCESS);
    2724: }

    2726: static PetscErrorCode DMNetworkDestroyComponentData(DM dm)
    2727: {
    2728:   DM_Network *network = (DM_Network *)dm->data;
    2729:   PetscInt    j, np;

    2731:   PetscFunctionBegin;
    2732:   if (network->header) {
    2733:     np = network->cloneshared->pEnd - network->cloneshared->pStart;
    2734:     for (j = 0; j < np; j++) {
    2735:       PetscCall(PetscFree5(network->header[j].size, network->header[j].key, network->header[j].offset, network->header[j].nvar, network->header[j].offsetvarrel));
    2736:       PetscCall(PetscFree(network->cvalue[j].data));
    2737:     }
    2738:     PetscCall(PetscFree2(network->header, network->cvalue));
    2739:   }
    2740:   PetscFunctionReturn(PETSC_SUCCESS);
    2741: }

    2743: PetscErrorCode DMDestroy_Network(DM dm)
    2744: {
    2745:   DM_Network *network = (DM_Network *)dm->data;
    2746:   PetscInt    j;

    2748:   PetscFunctionBegin;
    2749:   /*
    2750:     Developer Note: Due to the mixed topological definition of DMNetwork and data defined ON the
    2751:     network like DofSection, DataSection, *componentdataarray, and friends, when cloning, we share
    2752:     only the true topological data, and make our own data ON the network. Thus refct only refers
    2753:     to the number of references to topological data, and data ON the network is always destroyed.
    2754:     It is understood this is atypical for a DM, but is very intentional.
    2755:   */

    2757:   /* Always destroy data ON the network */
    2758:   PetscCall(PetscFree(network->Je));
    2759:   if (network->Jv) {
    2760:     PetscCall(PetscFree(network->Jvptr));
    2761:     PetscCall(PetscFree(network->Jv));
    2762:   }
    2763:   PetscCall(PetscSectionDestroy(&network->DataSection));
    2764:   PetscCall(PetscSectionDestroy(&network->DofSection));
    2765:   PetscCall(PetscFree(network->component));
    2766:   PetscCall(PetscFree(network->componentdataarray));
    2767:   PetscCall(DMNetworkDestroyComponentData(dm));

    2769:   PetscCall(DMDestroy(&network->plex)); /* this is cloned in DMClone_Network, so safe to destroy */

    2771:   /*
    2772:   Developer Note: The DMNetworkVertexInfo and DMNetworkEdgeInfo data structures are completely
    2773:   destroyed as they are again a mix of topological data:
    2774:     ISLocalToGlobalMapping            mapping;
    2775:     PetscSF                           sf;
    2776:   and data ON the network
    2777:     PetscSection                      DofSection;
    2778:     PetscSection                      GlobalDofSection;
    2779:   And the only way to access them currently is through DMNetworkAssembleGraphStructures which assembles
    2780:   everything. So we must destroy everything and require DMNetworkAssembleGraphStructures is called again
    2781:   for any clone.
    2782:   */
    2783:   PetscCall(ISLocalToGlobalMappingDestroy(&network->vertex.mapping));
    2784:   PetscCall(PetscSectionDestroy(&network->vertex.DofSection));
    2785:   PetscCall(PetscSectionDestroy(&network->vertex.GlobalDofSection));
    2786:   PetscCall(PetscSFDestroy(&network->vertex.sf));
    2787:   /* edge */
    2788:   PetscCall(ISLocalToGlobalMappingDestroy(&network->edge.mapping));
    2789:   PetscCall(PetscSectionDestroy(&network->edge.DofSection));
    2790:   PetscCall(PetscSectionDestroy(&network->edge.GlobalDofSection));
    2791:   PetscCall(PetscSFDestroy(&network->edge.sf));
    2792:   /* viewer options */
    2793:   PetscCall(ISDestroy(&network->vieweroptions.viewranks));
    2794:   /* Destroy the potentially cloneshared data */
    2795:   if (--network->cloneshared->refct <= 0) {
    2796:     /* Developer Note: I'm not sure if vltog can be reused or not, as I'm not sure what it's purpose is. I
    2797:      naively think it can be reused. */
    2798:     PetscCall(PetscFree(network->cloneshared->vltog));
    2799:     for (j = 0; j < network->cloneshared->Nsvtx; j++) PetscCall(PetscFree(network->cloneshared->svtx[j].sv));
    2800:     PetscCall(PetscFree(network->cloneshared->svtx));
    2801:     PetscCall(PetscFree2(network->cloneshared->subnetedge, network->cloneshared->subnetvtx));
    2802:     PetscCall(PetscHMapIDestroy(&network->cloneshared->svtable));
    2803:     PetscCall(PetscFree(network->cloneshared->subnet));
    2804:     PetscCall(PetscFree(network->cloneshared));
    2805:   }
    2806:   PetscCall(PetscFree(network)); /* Always freed as this structure is copied in a clone, not cloneshared */
    2807:   PetscFunctionReturn(PETSC_SUCCESS);
    2808: }

    2810: PetscErrorCode DMGlobalToLocalBegin_Network(DM dm, Vec g, InsertMode mode, Vec l)
    2811: {
    2812:   DM_Network *network = (DM_Network *)dm->data;

    2814:   PetscFunctionBegin;
    2815:   PetscCall(DMGlobalToLocalBegin(network->plex, g, mode, l));
    2816:   PetscFunctionReturn(PETSC_SUCCESS);
    2817: }

    2819: PetscErrorCode DMGlobalToLocalEnd_Network(DM dm, Vec g, InsertMode mode, Vec l)
    2820: {
    2821:   DM_Network *network = (DM_Network *)dm->data;

    2823:   PetscFunctionBegin;
    2824:   PetscCall(DMGlobalToLocalEnd(network->plex, g, mode, l));
    2825:   PetscFunctionReturn(PETSC_SUCCESS);
    2826: }

    2828: PetscErrorCode DMLocalToGlobalBegin_Network(DM dm, Vec l, InsertMode mode, Vec g)
    2829: {
    2830:   DM_Network *network = (DM_Network *)dm->data;

    2832:   PetscFunctionBegin;
    2833:   PetscCall(DMLocalToGlobalBegin(network->plex, l, mode, g));
    2834:   PetscFunctionReturn(PETSC_SUCCESS);
    2835: }

    2837: PetscErrorCode DMLocalToGlobalEnd_Network(DM dm, Vec l, InsertMode mode, Vec g)
    2838: {
    2839:   DM_Network *network = (DM_Network *)dm->data;

    2841:   PetscFunctionBegin;
    2842:   PetscCall(DMLocalToGlobalEnd(network->plex, l, mode, g));
    2843:   PetscFunctionReturn(PETSC_SUCCESS);
    2844: }

    2846: /*@
    2847:   DMNetworkGetVertexLocalToGlobalOrdering - Get vertex global index

    2849:   Not Collective

    2851:   Input Parameters:
    2852: + dm   - the `DMNETWORK` object
    2853: - vloc - local vertex ordering, start from 0

    2855:   Output Parameter:
    2856: . vg - global vertex ordering, start from 0

    2858:   Level: advanced

    2860: .seealso: `DM`, `DMNETWORK`, `DMNetworkSetVertexLocalToGlobalOrdering()`
    2861: @*/
    2862: PetscErrorCode DMNetworkGetVertexLocalToGlobalOrdering(DM dm, PetscInt vloc, PetscInt *vg)
    2863: {
    2864:   DM_Network *network = (DM_Network *)dm->data;
    2865:   PetscInt   *vltog   = network->cloneshared->vltog;

    2867:   PetscFunctionBegin;
    2868:   PetscCheck(vltog, PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONGSTATE, "Must call DMNetworkSetVertexLocalToGlobalOrdering() first");
    2869:   *vg = vltog[vloc];
    2870:   PetscFunctionReturn(PETSC_SUCCESS);
    2871: }

    2873: /*@
    2874:   DMNetworkSetVertexLocalToGlobalOrdering - Create and setup vertex local to global map

    2876:   Collective

    2878:   Input Parameters:
    2879: . dm - the `DMNETWORK` object

    2881:   Level: advanced

    2883: .seealso: `DM`, `DMNETWORK`, `DMNetworkGetGlobalVertexIndex()`
    2884: @*/
    2885: PetscErrorCode DMNetworkSetVertexLocalToGlobalOrdering(DM dm)
    2886: {
    2887:   DM_Network        *network = (DM_Network *)dm->data;
    2888:   MPI_Comm           comm;
    2889:   PetscMPIInt        rank, size, *displs = NULL, *recvcounts = NULL, remoterank;
    2890:   PetscBool          ghost;
    2891:   PetscInt          *vltog, nroots, nleaves, *vrange, k, N, lidx, ii;
    2892:   const PetscSFNode *iremote;
    2893:   PetscSF            vsf;
    2894:   Vec                Vleaves, Vleaves_seq;
    2895:   VecScatter         ctx;
    2896:   PetscScalar       *varr, val;
    2897:   const PetscScalar *varr_read;

    2899:   PetscFunctionBegin;
    2900:   PetscCall(PetscObjectGetComm((PetscObject)dm, &comm));
    2901:   PetscCallMPI(MPI_Comm_size(comm, &size));
    2902:   PetscCallMPI(MPI_Comm_rank(comm, &rank));

    2904:   if (size == 1) {
    2905:     nroots = network->cloneshared->vEnd - network->cloneshared->vStart;
    2906:     PetscCall(PetscMalloc1(nroots, &vltog));
    2907:     for (PetscInt i = 0; i < nroots; i++) vltog[i] = i;
    2908:     network->cloneshared->vltog = vltog;
    2909:     PetscFunctionReturn(PETSC_SUCCESS);
    2910:   }

    2912:   PetscCheck(network->cloneshared->distributecalled, comm, PETSC_ERR_ARG_WRONGSTATE, "Must call DMNetworkDistribute() first");
    2913:   if (network->cloneshared->vltog) PetscCall(PetscFree(network->cloneshared->vltog));

    2915:   PetscCall(DMNetworkSetSubMap_private(dm, network->cloneshared->vStart, network->cloneshared->vEnd, &network->vertex.mapping));
    2916:   PetscCall(PetscSFGetSubSF(network->plex->sf, network->vertex.mapping, &network->vertex.sf));
    2917:   vsf = network->vertex.sf;

    2919:   PetscCall(PetscMalloc3(size + 1, &vrange, size, &displs, size, &recvcounts));
    2920:   PetscCall(PetscSFGetGraph(vsf, &nroots, &nleaves, NULL, &iremote));

    2922:   for (PetscMPIInt i = 0; i < size; i++) {
    2923:     displs[i]     = i;
    2924:     recvcounts[i] = 1;
    2925:   }

    2927:   ii        = nroots - nleaves; /* local number of vertices, excluding ghosts */
    2928:   vrange[0] = 0;
    2929:   PetscCallMPI(MPI_Allgatherv(&ii, 1, MPIU_INT, vrange + 1, recvcounts, displs, MPIU_INT, comm));
    2930:   for (PetscMPIInt i = 2; i < size + 1; i++) vrange[i] += vrange[i - 1];

    2932:   PetscCall(PetscMalloc1(nroots, &vltog));
    2933:   network->cloneshared->vltog = vltog;

    2935:   /* Set vltog for non-ghost vertices */
    2936:   k = 0;
    2937:   for (PetscInt i = 0; i < nroots; i++) {
    2938:     PetscCall(DMNetworkIsGhostVertex(dm, i + network->cloneshared->vStart, &ghost));
    2939:     if (ghost) continue;
    2940:     vltog[i] = vrange[rank] + k++;
    2941:   }
    2942:   PetscCall(PetscFree3(vrange, displs, recvcounts));

    2944:   /* Set vltog for ghost vertices */
    2945:   /* (a) create parallel Vleaves and sequential Vleaves_seq to convert local iremote[*].index to global index */
    2946:   PetscCall(VecCreate(comm, &Vleaves));
    2947:   PetscCall(VecSetSizes(Vleaves, 2 * nleaves, PETSC_DETERMINE));
    2948:   PetscCall(VecSetFromOptions(Vleaves));
    2949:   PetscCall(VecGetArray(Vleaves, &varr));
    2950:   for (PetscInt i = 0; i < nleaves; i++) {
    2951:     varr[2 * i]     = (PetscScalar)iremote[i].rank;  /* rank of remote process */
    2952:     varr[2 * i + 1] = (PetscScalar)iremote[i].index; /* local index in remote process */
    2953:   }
    2954:   PetscCall(VecRestoreArray(Vleaves, &varr));

    2956:   /* (b) scatter local info to remote processes via VecScatter() */
    2957:   PetscCall(VecScatterCreateToAll(Vleaves, &ctx, &Vleaves_seq));
    2958:   PetscCall(VecScatterBegin(ctx, Vleaves, Vleaves_seq, INSERT_VALUES, SCATTER_FORWARD));
    2959:   PetscCall(VecScatterEnd(ctx, Vleaves, Vleaves_seq, INSERT_VALUES, SCATTER_FORWARD));

    2961:   /* (c) convert local indices to global indices in parallel vector Vleaves */
    2962:   PetscCall(VecGetSize(Vleaves_seq, &N));
    2963:   PetscCall(VecGetArrayRead(Vleaves_seq, &varr_read));
    2964:   for (PetscInt i = 0; i < N; i += 2) {
    2965:     remoterank = (PetscMPIInt)PetscRealPart(varr_read[i]);
    2966:     if (remoterank == rank) {
    2967:       k    = i + 1; /* row number */
    2968:       lidx = (PetscInt)PetscRealPart(varr_read[i + 1]);
    2969:       val  = (PetscScalar)vltog[lidx]; /* global index for non-ghost vertex computed above */
    2970:       PetscCall(VecSetValues(Vleaves, 1, &k, &val, INSERT_VALUES));
    2971:     }
    2972:   }
    2973:   PetscCall(VecRestoreArrayRead(Vleaves_seq, &varr_read));
    2974:   PetscCall(VecAssemblyBegin(Vleaves));
    2975:   PetscCall(VecAssemblyEnd(Vleaves));

    2977:   /* (d) Set vltog for ghost vertices by copying local values of Vleaves */
    2978:   PetscCall(VecGetArrayRead(Vleaves, &varr_read));
    2979:   k = 0;
    2980:   for (PetscInt i = 0; i < nroots; i++) {
    2981:     PetscCall(DMNetworkIsGhostVertex(dm, i + network->cloneshared->vStart, &ghost));
    2982:     if (!ghost) continue;
    2983:     vltog[i] = (PetscInt)PetscRealPart(varr_read[2 * k + 1]);
    2984:     k++;
    2985:   }
    2986:   PetscCall(VecRestoreArrayRead(Vleaves, &varr_read));

    2988:   PetscCall(VecDestroy(&Vleaves));
    2989:   PetscCall(VecDestroy(&Vleaves_seq));
    2990:   PetscCall(VecScatterDestroy(&ctx));
    2991:   PetscFunctionReturn(PETSC_SUCCESS);
    2992: }

    2994: static inline PetscErrorCode DMISAddSize_private(DM_Network *network, PetscInt p, PetscInt numkeys, PetscInt keys[], PetscInt blocksize[], PetscInt nselectedvar[], PetscInt *nidx)
    2995: {
    2996:   PetscInt                 i, j, ncomps, nvar, key, offset = 0;
    2997:   DMNetworkComponentHeader header;

    2999:   PetscFunctionBegin;
    3000:   PetscCall(PetscSectionGetOffset(network->DataSection, p, &offset));
    3001:   ncomps = ((DMNetworkComponentHeader)(network->componentdataarray + offset))->ndata;
    3002:   header = (DMNetworkComponentHeader)(network->componentdataarray + offset);

    3004:   for (i = 0; i < ncomps; i++) {
    3005:     key  = header->key[i];
    3006:     nvar = header->nvar[i];
    3007:     for (j = 0; j < numkeys; j++) {
    3008:       if (key == keys[j]) {
    3009:         if (!blocksize || blocksize[j] == -1) {
    3010:           *nidx += nvar;
    3011:         } else {
    3012:           *nidx += nselectedvar[j] * nvar / blocksize[j];
    3013:         }
    3014:       }
    3015:     }
    3016:   }
    3017:   PetscFunctionReturn(PETSC_SUCCESS);
    3018: }

    3020: static inline PetscErrorCode DMISComputeIdx_private(DM dm, PetscInt p, PetscInt numkeys, PetscInt keys[], PetscInt blocksize[], PetscInt nselectedvar[], PetscInt *selectedvar[], PetscInt *ii, PetscInt *idx)
    3021: {
    3022:   PetscInt                 i, j, ncomps, nvar, key, offsetg, k, k1, offset = 0;
    3023:   DM_Network              *network = (DM_Network *)dm->data;
    3024:   DMNetworkComponentHeader header;

    3026:   PetscFunctionBegin;
    3027:   PetscCall(PetscSectionGetOffset(network->DataSection, p, &offset));
    3028:   ncomps = ((DMNetworkComponentHeader)(network->componentdataarray + offset))->ndata;
    3029:   header = (DMNetworkComponentHeader)(network->componentdataarray + offset);

    3031:   for (i = 0; i < ncomps; i++) {
    3032:     key  = header->key[i];
    3033:     nvar = header->nvar[i];
    3034:     for (j = 0; j < numkeys; j++) {
    3035:       if (key != keys[j]) continue;

    3037:       PetscCall(DMNetworkGetGlobalVecOffset(dm, p, i, &offsetg));
    3038:       if (!blocksize || blocksize[j] == -1) {
    3039:         for (k = 0; k < nvar; k++) idx[(*ii)++] = offsetg + k;
    3040:       } else {
    3041:         for (k = 0; k < nvar; k += blocksize[j]) {
    3042:           for (k1 = 0; k1 < nselectedvar[j]; k1++) idx[(*ii)++] = offsetg + k + selectedvar[j][k1];
    3043:         }
    3044:       }
    3045:     }
    3046:   }
    3047:   PetscFunctionReturn(PETSC_SUCCESS);
    3048: }

    3050: /*@
    3051:   DMNetworkCreateIS - Create an index set object from the global vector of the network

    3053:   Collective

    3055:   Input Parameters:
    3056: + dm           - `DMNETWORK` object
    3057: . numkeys      - number of keys
    3058: . keys         - array of keys that define the components of the variables you wish to extract
    3059: . blocksize    - block size of the variables associated to the component
    3060: . nselectedvar - number of variables in each block to select
    3061: - selectedvar  - the offset into the block of each variable in each block to select

    3063:   Output Parameter:
    3064: . is - the index set

    3066:   Level: advanced

    3068:   Notes:
    3069:   Use blocksize[i] of -1 to indicate select all the variables of the i-th component, for which nselectvar[i] and selectedvar[i] are ignored. Use` NULL`, `NULL`, `NULL` to indicate for all selected components one wishes to obtain all the values of that component. For example, `DMNetworkCreateIS`(dm,1,&key,NULL,NULL,NULL,&is) will return an is that extracts all the variables for the 0-th component.

    3071: .seealso: `DM`, `DMNETWORK`, `DMNetworkCreate()`, `ISCreateGeneral()`, `DMNetworkCreateLocalIS()`
    3072: @*/
    3073: PetscErrorCode DMNetworkCreateIS(DM dm, PetscInt numkeys, PetscInt keys[], PetscInt blocksize[], PetscInt nselectedvar[], PetscInt *selectedvar[], IS *is)
    3074: {
    3075:   MPI_Comm    comm;
    3076:   DM_Network *network = (DM_Network *)dm->data;
    3077:   PetscInt    i, p, estart, eend, vstart, vend, nidx, *idx;
    3078:   PetscBool   ghost;

    3080:   PetscFunctionBegin;
    3081:   PetscCall(PetscObjectGetComm((PetscObject)dm, &comm));

    3083:   /* Check input parameters */
    3084:   for (i = 0; i < numkeys; i++) {
    3085:     if (!blocksize || blocksize[i] == -1) continue;
    3086:     PetscCheck(nselectedvar[i] <= blocksize[i], PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "number of selectedvariables %" PetscInt_FMT " cannot be larger than blocksize %" PetscInt_FMT, nselectedvar[i], blocksize[i]);
    3087:   }

    3089:   PetscCall(DMNetworkGetEdgeRange(dm, &estart, &eend));
    3090:   PetscCall(DMNetworkGetVertexRange(dm, &vstart, &vend));

    3092:   /* Get local number of idx */
    3093:   nidx = 0;
    3094:   for (p = estart; p < eend; p++) PetscCall(DMISAddSize_private(network, p, numkeys, keys, blocksize, nselectedvar, &nidx));
    3095:   for (p = vstart; p < vend; p++) {
    3096:     PetscCall(DMNetworkIsGhostVertex(dm, p, &ghost));
    3097:     if (ghost) continue;
    3098:     PetscCall(DMISAddSize_private(network, p, numkeys, keys, blocksize, nselectedvar, &nidx));
    3099:   }

    3101:   /* Compute idx */
    3102:   PetscCall(PetscMalloc1(nidx, &idx));
    3103:   i = 0;
    3104:   for (p = estart; p < eend; p++) PetscCall(DMISComputeIdx_private(dm, p, numkeys, keys, blocksize, nselectedvar, selectedvar, &i, idx));
    3105:   for (p = vstart; p < vend; p++) {
    3106:     PetscCall(DMNetworkIsGhostVertex(dm, p, &ghost));
    3107:     if (ghost) continue;
    3108:     PetscCall(DMISComputeIdx_private(dm, p, numkeys, keys, blocksize, nselectedvar, selectedvar, &i, idx));
    3109:   }

    3111:   /* Create is */
    3112:   PetscCall(ISCreateGeneral(comm, nidx, idx, PETSC_COPY_VALUES, is));
    3113:   PetscCall(PetscFree(idx));
    3114:   PetscFunctionReturn(PETSC_SUCCESS);
    3115: }

    3117: static inline PetscErrorCode DMISComputeLocalIdx_private(DM dm, PetscInt p, PetscInt numkeys, PetscInt keys[], PetscInt blocksize[], PetscInt nselectedvar[], PetscInt *selectedvar[], PetscInt *ii, PetscInt *idx)
    3118: {
    3119:   PetscInt                 i, j, ncomps, nvar, key, offsetl, k, k1, offset = 0;
    3120:   DM_Network              *network = (DM_Network *)dm->data;
    3121:   DMNetworkComponentHeader header;

    3123:   PetscFunctionBegin;
    3124:   PetscCall(PetscSectionGetOffset(network->DataSection, p, &offset));
    3125:   ncomps = ((DMNetworkComponentHeader)(network->componentdataarray + offset))->ndata;
    3126:   header = (DMNetworkComponentHeader)(network->componentdataarray + offset);

    3128:   for (i = 0; i < ncomps; i++) {
    3129:     key  = header->key[i];
    3130:     nvar = header->nvar[i];
    3131:     for (j = 0; j < numkeys; j++) {
    3132:       if (key != keys[j]) continue;

    3134:       PetscCall(DMNetworkGetLocalVecOffset(dm, p, i, &offsetl));
    3135:       if (!blocksize || blocksize[j] == -1) {
    3136:         for (k = 0; k < nvar; k++) idx[(*ii)++] = offsetl + k;
    3137:       } else {
    3138:         for (k = 0; k < nvar; k += blocksize[j]) {
    3139:           for (k1 = 0; k1 < nselectedvar[j]; k1++) idx[(*ii)++] = offsetl + k + selectedvar[j][k1];
    3140:         }
    3141:       }
    3142:     }
    3143:   }
    3144:   PetscFunctionReturn(PETSC_SUCCESS);
    3145: }

    3147: /*@
    3148:   DMNetworkCreateLocalIS - Create an index set object from the local vector of the network

    3150:   Not Collective

    3152:   Input Parameters:
    3153: + dm           - `DMNETWORK` object
    3154: . numkeys      - number of keys
    3155: . keys         - array of keys that define the components of the variables you wish to extract
    3156: . blocksize    - block size of the variables associated to the component
    3157: . nselectedvar - number of variables in each block to select
    3158: - selectedvar  - the offset into the block of each variable in each block to select

    3160:   Output Parameter:
    3161: . is - the index set

    3163:   Level: advanced

    3165:   Notes:
    3166:   Use blocksize[i] of -1 to indicate select all the variables of the i-th component, for which nselectvar[i] and selectedvar[i] are ignored. Use `NULL`, `NULL`, `NULL` to indicate for all selected components one wishes to obtain all the values of that component. For example, `DMNetworkCreateLocalIS`(dm,1,&key,`NULL`,`NULL`,`NULL`,&is) will return an is that extracts all the variables for the 0-th component.

    3168: .seealso: `DM`, `DMNETWORK`, `DMNetworkCreate()`, `DMNetworkCreateIS()`, `ISCreateGeneral()`
    3169: @*/
    3170: PetscErrorCode DMNetworkCreateLocalIS(DM dm, PetscInt numkeys, PetscInt keys[], PetscInt blocksize[], PetscInt nselectedvar[], PetscInt *selectedvar[], IS *is)
    3171: {
    3172:   DM_Network *network = (DM_Network *)dm->data;
    3173:   PetscInt    i, p, pstart, pend, nidx, *idx;

    3175:   PetscFunctionBegin;
    3176:   /* Check input parameters */
    3177:   for (i = 0; i < numkeys; i++) {
    3178:     if (!blocksize || blocksize[i] == -1) continue;
    3179:     PetscCheck(nselectedvar[i] <= blocksize[i], PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "number of selectedvariables %" PetscInt_FMT " cannot be larger than blocksize %" PetscInt_FMT, nselectedvar[i], blocksize[i]);
    3180:   }

    3182:   pstart = network->cloneshared->pStart;
    3183:   pend   = network->cloneshared->pEnd;

    3185:   /* Get local number of idx */
    3186:   nidx = 0;
    3187:   for (p = pstart; p < pend; p++) PetscCall(DMISAddSize_private(network, p, numkeys, keys, blocksize, nselectedvar, &nidx));

    3189:   /* Compute local idx */
    3190:   PetscCall(PetscMalloc1(nidx, &idx));
    3191:   i = 0;
    3192:   for (p = pstart; p < pend; p++) PetscCall(DMISComputeLocalIdx_private(dm, p, numkeys, keys, blocksize, nselectedvar, selectedvar, &i, idx));

    3194:   /* Create is */
    3195:   PetscCall(ISCreateGeneral(PETSC_COMM_SELF, nidx, idx, PETSC_COPY_VALUES, is));
    3196:   PetscCall(PetscFree(idx));
    3197:   PetscFunctionReturn(PETSC_SUCCESS);
    3198: }
    3199: /*@
    3200:   DMNetworkFinalizeComponents - Sets up internal data structures for the sections and components. It is called after registering new components and adding all components
    3201:   to the cloned network. After calling this subroutine, no new components can be added to the network.

    3203:   Collective

    3205:   Input Parameter:
    3206: . dm - the `DMNETWORK` object

    3208:   Level: beginner

    3210: .seealso: `DM`, `DMNETWORK`, `DMNetworkAddComponent()`, `DMNetworkRegisterComponent()`, `DMSetUp()`
    3211: @*/
    3212: PetscErrorCode DMNetworkFinalizeComponents(DM dm)
    3213: {
    3214:   DM_Network *network = (DM_Network *)dm->data;

    3216:   PetscFunctionBegin;
    3217:   if (network->componentsetup) PetscFunctionReturn(PETSC_SUCCESS);
    3218:   PetscCall(DMNetworkComponentSetUp(dm));
    3219:   PetscCall(DMNetworkVariablesSetUp(dm));
    3220:   PetscCall(DMSetLocalSection(network->plex, network->DofSection));
    3221:   PetscCall(DMGetGlobalSection(network->plex, &network->GlobalDofSection));
    3222:   network->componentsetup = PETSC_TRUE;
    3223:   PetscFunctionReturn(PETSC_SUCCESS);
    3224: }