Date: 2025-03-29T05:58:25+00:00

Actual source code: ex14.c

      1: static char help[] = "Tests DMCreateDomainDecomposition.\n\n";

      3: /*
      4: Use the options
      5:      -da_grid_x <nx> - number of grid points in x direction, if M < 0
      6:      -da_grid_y <ny> - number of grid points in y direction, if N < 0
      7:      -da_processors_x <MX> number of processors in x directio
      8:      -da_processors_y <MY> number of processors in x direction
      9: */

     11: #include <petscdm.h>
     12: #include <petscdmda.h>

     14: PetscErrorCode FillLocalSubdomain(DM da, Vec gvec)
     15: {
     16:   DMDALocalInfo info;
     17:   PetscMPIInt   rank;
     18:   PetscInt      i, j, k, l;

     20:   PetscFunctionBeginUser;
     21:   PetscCallMPI(MPI_Comm_rank(PETSC_COMM_WORLD, &rank));
     22:   PetscCall(DMDAGetLocalInfo(da, &info));

     24:   if (info.dim == 3) {
     25:     PetscScalar ***g;
     26:     PetscCall(DMDAVecGetArray(da, gvec, &g));
     27:     /* loop over ghosts */
     28:     for (k = info.zs; k < info.zs + info.zm; k++) {
     29:       for (j = info.ys; j < info.ys + info.ym; j++) {
     30:         for (i = info.xs; i < info.xs + info.xm; i++) {
     31:           g[k][j][info.dof * i + 0] = i;
     32:           g[k][j][info.dof * i + 1] = j;
     33:           g[k][j][info.dof * i + 2] = k;
     34:         }
     35:       }
     36:     }
     37:     PetscCall(DMDAVecRestoreArray(da, gvec, &g));
     38:   }
     39:   if (info.dim == 2) {
     40:     PetscScalar **g;
     41:     PetscCall(DMDAVecGetArray(da, gvec, &g));
     42:     /* loop over ghosts */
     43:     for (j = info.ys; j < info.ys + info.ym; j++) {
     44:       for (i = info.xs; i < info.xs + info.xm; i++) {
     45:         for (l = 0; l < info.dof; l++) {
     46:           g[j][info.dof * i + 0] = i;
     47:           g[j][info.dof * i + 1] = j;
     48:           g[j][info.dof * i + 2] = rank;
     49:         }
     50:       }
     51:     }
     52:     PetscCall(DMDAVecRestoreArray(da, gvec, &g));
     53:   }
     54:   PetscFunctionReturn(PETSC_SUCCESS);
     55: }

     57: int main(int argc, char **argv)
     58: {
     59:   DM            da, *subda;
     60:   PetscInt      i, dim = 3, dof = 3;
     61:   PetscInt      M = 25, N = 25, P = 25;
     62:   PetscMPIInt   size, rank;
     63:   Vec           v;
     64:   Vec           slvec, sgvec;
     65:   IS           *ois, *iis;
     66:   VecScatter    oscata;
     67:   VecScatter   *iscat, *oscat, *gscat;
     68:   DMDALocalInfo info;
     69:   PetscBool     patchis_offproc = PETSC_TRUE;

     71:   PetscFunctionBeginUser;
     72:   PetscCall(PetscInitialize(&argc, &argv, NULL, help));
     73:   PetscCall(PetscOptionsGetInt(NULL, NULL, "-dim", &dim, NULL));
     74:   PetscCall(PetscOptionsGetInt(NULL, NULL, "-dof", &dof, NULL));

     76:   /* Create distributed array and get vectors */
     77:   PetscCallMPI(MPI_Comm_size(PETSC_COMM_WORLD, &size));
     78:   PetscCallMPI(MPI_Comm_rank(PETSC_COMM_WORLD, &rank));
     79:   if (dim == 2) {
     80:     PetscCall(DMDACreate2d(PETSC_COMM_WORLD, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DMDA_STENCIL_STAR, M, N, PETSC_DECIDE, PETSC_DECIDE, dof, 1, NULL, NULL, &da));
     81:   } else if (dim == 3) {
     82:     PetscCall(DMDACreate3d(PETSC_COMM_WORLD, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DMDA_STENCIL_STAR, M, N, P, PETSC_DECIDE, PETSC_DECIDE, PETSC_DECIDE, dof, 1, NULL, NULL, NULL, &da));
     83:   }
     84:   PetscCall(DMSetFromOptions(da));
     85:   PetscCall(DMSetUp(da));
     86:   PetscCall(DMDAGetLocalInfo(da, &info));

     88:   PetscCall(DMCreateDomainDecomposition(da, NULL, NULL, &iis, &ois, &subda));
     89:   PetscCall(DMCreateDomainDecompositionScatters(da, 1, subda, &iscat, &oscat, &gscat));

     91:   /* TODO: broken for dim=2 */
     92:   if (dim == 3) {
     93:     DMDALocalInfo subinfo;
     94:     MatStencil    lower, upper;
     95:     IS            patchis;
     96:     Vec           smallvec;
     97:     Vec           largevec;
     98:     VecScatter    patchscat;

    100:     PetscCall(DMDAGetLocalInfo(subda[0], &subinfo));

    102:     lower.i = info.xs;
    103:     lower.j = info.ys;
    104:     lower.k = info.zs;
    105:     upper.i = info.xs + info.xm;
    106:     upper.j = info.ys + info.ym;
    107:     upper.k = info.zs + info.zm;

    109:     /* test the patch IS as a thing to scatter to/from */
    110:     PetscCall(DMDACreatePatchIS(da, &lower, &upper, &patchis, patchis_offproc));
    111:     PetscCall(DMGetGlobalVector(da, &largevec));

    113:     PetscCall(VecCreate(PETSC_COMM_SELF, &smallvec));
    114:     PetscCall(VecSetSizes(smallvec, info.dof * (upper.i - lower.i) * (upper.j - lower.j) * (upper.k - lower.k), PETSC_DECIDE));
    115:     PetscCall(VecSetFromOptions(smallvec));
    116:     PetscCall(VecScatterCreate(smallvec, NULL, largevec, patchis, &patchscat));

    118:     PetscCall(FillLocalSubdomain(subda[0], smallvec));
    119:     PetscCall(VecSet(largevec, 0));

    121:     PetscCall(VecScatterBegin(patchscat, smallvec, largevec, ADD_VALUES, SCATTER_FORWARD));
    122:     PetscCall(VecScatterEnd(patchscat, smallvec, largevec, ADD_VALUES, SCATTER_FORWARD));
    123:     PetscCall(ISView(patchis, PETSC_VIEWER_STDOUT_WORLD));
    124:     PetscCall(VecScatterView(patchscat, PETSC_VIEWER_STDOUT_WORLD));

    126:     for (i = 0; i < size; i++) {
    127:       if (i == rank) PetscCall(VecView(smallvec, PETSC_VIEWER_STDOUT_SELF));
    128:       PetscCallMPI(MPI_Barrier(PETSC_COMM_WORLD));
    129:     }

    131:     PetscCallMPI(MPI_Barrier(PETSC_COMM_WORLD));
    132:     PetscCall(VecView(largevec, PETSC_VIEWER_STDOUT_WORLD));

    134:     PetscCall(VecDestroy(&smallvec));
    135:     PetscCall(DMRestoreGlobalVector(da, &largevec));
    136:     PetscCall(ISDestroy(&patchis));
    137:     PetscCall(VecScatterDestroy(&patchscat));
    138:   }

    140:   /* view the various parts */
    141:   {
    142:     for (i = 0; i < size; i++) {
    143:       if (i == rank) {
    144:         PetscCall(PetscPrintf(PETSC_COMM_SELF, "Processor %d: \n", i));
    145:         PetscCall(DMView(subda[0], PETSC_VIEWER_STDOUT_SELF));
    146:       }
    147:       PetscCallMPI(MPI_Barrier(PETSC_COMM_WORLD));
    148:     }

    150:     PetscCall(DMGetLocalVector(subda[0], &slvec));
    151:     PetscCall(DMGetGlobalVector(subda[0], &sgvec));
    152:     PetscCall(DMGetGlobalVector(da, &v));

    154:     /* test filling outer between the big DM and the small ones with the IS scatter*/
    155:     PetscCall(VecScatterCreate(v, ois[0], sgvec, NULL, &oscata));

    157:     PetscCall(FillLocalSubdomain(subda[0], sgvec));

    159:     PetscCall(VecScatterBegin(oscata, sgvec, v, ADD_VALUES, SCATTER_REVERSE));
    160:     PetscCall(VecScatterEnd(oscata, sgvec, v, ADD_VALUES, SCATTER_REVERSE));

    162:     /* test the local-to-local scatter */

    164:     /* fill up the local subdomain and then add them together */
    165:     PetscCall(FillLocalSubdomain(da, v));

    167:     PetscCall(VecScatterBegin(gscat[0], v, slvec, ADD_VALUES, SCATTER_FORWARD));
    168:     PetscCall(VecScatterEnd(gscat[0], v, slvec, ADD_VALUES, SCATTER_FORWARD));

    170:     PetscCall(VecView(v, PETSC_VIEWER_STDOUT_WORLD));

    172:     /* test ghost scattering backwards */

    174:     PetscCall(VecSet(v, 0));

    176:     PetscCall(VecScatterBegin(gscat[0], slvec, v, ADD_VALUES, SCATTER_REVERSE));
    177:     PetscCall(VecScatterEnd(gscat[0], slvec, v, ADD_VALUES, SCATTER_REVERSE));

    179:     PetscCall(VecView(v, PETSC_VIEWER_STDOUT_WORLD));

    181:     /* test overlap scattering backwards */

    183:     PetscCall(DMLocalToGlobalBegin(subda[0], slvec, ADD_VALUES, sgvec));
    184:     PetscCall(DMLocalToGlobalEnd(subda[0], slvec, ADD_VALUES, sgvec));

    186:     PetscCall(VecSet(v, 0));

    188:     PetscCall(VecScatterBegin(oscat[0], sgvec, v, ADD_VALUES, SCATTER_REVERSE));
    189:     PetscCall(VecScatterEnd(oscat[0], sgvec, v, ADD_VALUES, SCATTER_REVERSE));

    191:     PetscCall(VecView(v, PETSC_VIEWER_STDOUT_WORLD));

    193:     /* test interior scattering backwards */

    195:     PetscCall(VecSet(v, 0));

    197:     PetscCall(VecScatterBegin(iscat[0], sgvec, v, ADD_VALUES, SCATTER_REVERSE));
    198:     PetscCall(VecScatterEnd(iscat[0], sgvec, v, ADD_VALUES, SCATTER_REVERSE));

    200:     PetscCall(VecView(v, PETSC_VIEWER_STDOUT_WORLD));

    202:     /* test matrix allocation */
    203:     for (i = 0; i < size; i++) {
    204:       if (i == rank) {
    205:         Mat m;
    206:         PetscCall(PetscPrintf(PETSC_COMM_SELF, "Processor %d: \n", i));
    207:         PetscCall(DMSetMatType(subda[0], MATAIJ));
    208:         PetscCall(DMCreateMatrix(subda[0], &m));
    209:         PetscCall(MatView(m, PETSC_VIEWER_STDOUT_SELF));
    210:         PetscCall(MatDestroy(&m));
    211:       }
    212:       PetscCallMPI(MPI_Barrier(PETSC_COMM_WORLD));
    213:     }
    214:     PetscCall(DMRestoreLocalVector(subda[0], &slvec));
    215:     PetscCall(DMRestoreGlobalVector(subda[0], &sgvec));
    216:     PetscCall(DMRestoreGlobalVector(da, &v));
    217:   }

    219:   PetscCall(DMDestroy(&subda[0]));
    220:   PetscCall(ISDestroy(&ois[0]));
    221:   PetscCall(ISDestroy(&iis[0]));

    223:   PetscCall(VecScatterDestroy(&iscat[0]));
    224:   PetscCall(VecScatterDestroy(&oscat[0]));
    225:   PetscCall(VecScatterDestroy(&gscat[0]));
    226:   PetscCall(VecScatterDestroy(&oscata));

    228:   PetscCall(PetscFree(iscat));
    229:   PetscCall(PetscFree(oscat));
    230:   PetscCall(PetscFree(gscat));
    231:   PetscCall(PetscFree(oscata));

    233:   PetscCall(PetscFree(subda));
    234:   PetscCall(PetscFree(ois));
    235:   PetscCall(PetscFree(iis));

    237:   PetscCall(DMDestroy(&da));
    238:   PetscCall(PetscFinalize());
    239:   return 0;
    240: }