Date: 2022-03-31T04:48:34+00:00

Actual source code: ex56.c

      1: static char help[] = "3D, tri-quadratic hexahedra (Q1), displacement finite element formulation\n\
      2: of linear elasticity.  E=1.0, nu=1/3.\n\
      3: Unit cube domain with Dirichlet boundary\n\n";

      5: #include <petscdmplex.h>
      6: #include <petscsnes.h>
      7: #include <petscds.h>
      8: #include <petscdmforest.h>

     10: static PetscReal s_soft_alpha=1.e-3;
     11: static PetscReal s_mu=0.4;
     12: static PetscReal s_lambda=0.4;

     14: static void f0_bd_u_3d(PetscInt dim, PetscInt Nf, PetscInt NfAux,
     15:                        const PetscInt uOff[], const PetscInt uOff_x[], const PetscScalar u[], const PetscScalar u_t[], const PetscScalar u_x[],
     16:                        const PetscInt aOff[], const PetscInt aOff_x[], const PetscScalar a[], const PetscScalar a_t[], const PetscScalar a_x[],
     17:                        PetscReal t, const PetscReal x[], const PetscReal n[], PetscInt numConstants, const PetscScalar constants[], PetscScalar f0[])
     18: {
     19:   f0[0] = 1;     /* x direction pull */
     20:   f0[1] = -x[2]; /* add a twist around x-axis */
     21:   f0[2] =  x[1];
     22: }

     24: static void f1_bd_u(PetscInt dim, PetscInt Nf, PetscInt NfAux,
     25:                     const PetscInt uOff[], const PetscInt uOff_x[], const PetscScalar u[], const PetscScalar u_t[], const PetscScalar u_x[],
     26:                     const PetscInt aOff[], const PetscInt aOff_x[], const PetscScalar a[], const PetscScalar a_t[], const PetscScalar a_x[],
     27:                     PetscReal t, const PetscReal x[], const PetscReal n[], PetscInt numConstants, const PetscScalar constants[], PetscScalar f1[])
     28: {
     29:   const PetscInt Ncomp = dim;
     30:   PetscInt       comp, d;
     31:   for (comp = 0; comp < Ncomp; ++comp) {
     32:     for (d = 0; d < dim; ++d) {
     33:       f1[comp*dim+d] = 0.0;
     34:     }
     35:   }
     36: }

     38: /* gradU[comp*dim+d] = {u_x, u_y} or {u_x, u_y, u_z} */
     39: static void f1_u_3d_alpha(PetscInt dim, PetscInt Nf, PetscInt NfAux,
     40:                           const PetscInt uOff[], const PetscInt uOff_x[], const PetscScalar u[], const PetscScalar u_t[], const PetscScalar u_x[],
     41:                           const PetscInt aOff[], const PetscInt aOff_x[], const PetscScalar a[], const PetscScalar a_t[], const PetscScalar a_x[],
     42:                           PetscReal t, const PetscReal x[], PetscInt numConstants, const PetscScalar constants[], PetscScalar f1[])
     43: {
     44:   PetscReal trace,mu=s_mu,lambda=s_lambda,rad;
     45:   PetscInt i,j;
     46:   for (i=0,rad=0.;i<dim;i++) {
     47:     PetscReal t=x[i];
     48:     rad += t*t;
     49:   }
     50:   rad = PetscSqrtReal(rad);
     51:   if (rad>0.25) {
     52:     mu *= s_soft_alpha;
     53:     lambda *= s_soft_alpha; /* we could keep the bulk the same like rubberish */
     54:   }
     55:   for (i=0,trace=0; i < dim; ++i) {
     56:     trace += PetscRealPart(u_x[i*dim+i]);
     57:   }
     58:   for (i=0; i < dim; ++i) {
     59:     for (j=0; j < dim; ++j) {
     60:       f1[i*dim+j] = mu*(u_x[i*dim+j]+u_x[j*dim+i]);
     61:     }
     62:     f1[i*dim+i] += lambda*trace;
     63:   }
     64: }

     66: /* gradU[comp*dim+d] = {u_x, u_y} or {u_x, u_y, u_z} */
     67: static void f1_u_3d(PetscInt dim, PetscInt Nf, PetscInt NfAux,
     68:                     const PetscInt uOff[], const PetscInt uOff_x[], const PetscScalar u[], const PetscScalar u_t[], const PetscScalar u_x[],
     69:                     const PetscInt aOff[], const PetscInt aOff_x[], const PetscScalar a[], const PetscScalar a_t[], const PetscScalar a_x[],
     70:                     PetscReal t, const PetscReal x[], PetscInt numConstants, const PetscScalar constants[], PetscScalar f1[])
     71: {
     72:   PetscReal trace,mu=s_mu,lambda=s_lambda;
     73:   PetscInt i,j;
     74:   for (i=0,trace=0; i < dim; ++i) {
     75:     trace += PetscRealPart(u_x[i*dim+i]);
     76:   }
     77:   for (i=0; i < dim; ++i) {
     78:     for (j=0; j < dim; ++j) {
     79:       f1[i*dim+j] = mu*(u_x[i*dim+j]+u_x[j*dim+i]);
     80:     }
     81:     f1[i*dim+i] += lambda*trace;
     82:   }
     83: }

     85: /* 3D elasticity */
     86: #define IDX(ii,jj,kk,ll) (27*ii+9*jj+3*kk+ll)

     88: void g3_uu_3d_private( PetscScalar g3[], const PetscReal mu, const PetscReal lambda)
     89: {
     90:   if (1) {
     91:     g3[0] += lambda;
     92:     g3[0] += mu;
     93:     g3[0] += mu;
     94:     g3[4] += lambda;
     95:     g3[8] += lambda;
     96:     g3[10] += mu;
     97:     g3[12] += mu;
     98:     g3[20] += mu;
     99:     g3[24] += mu;
    100:     g3[28] += mu;
    101:     g3[30] += mu;
    102:     g3[36] += lambda;
    103:     g3[40] += lambda;
    104:     g3[40] += mu;
    105:     g3[40] += mu;
    106:     g3[44] += lambda;
    107:     g3[50] += mu;
    108:     g3[52] += mu;
    109:     g3[56] += mu;
    110:     g3[60] += mu;
    111:     g3[68] += mu;
    112:     g3[70] += mu;
    113:     g3[72] += lambda;
    114:     g3[76] += lambda;
    115:     g3[80] += lambda;
    116:     g3[80] += mu;
    117:     g3[80] += mu;
    118:   } else {
    119:     int        i,j,k,l;
    120:     static int cc=-1;
    121:     cc++;
    122:     for (i = 0; i < 3; ++i) {
    123:       for (j = 0; j < 3; ++j) {
    124:         for (k = 0; k < 3; ++k) {
    125:           for (l = 0; l < 3; ++l) {
    126:             if (k==l && i==j) g3[IDX(i,j,k,l)] += lambda;
    127:             if (i==k && j==l) g3[IDX(i,j,k,l)] += mu;
    128:             if (i==l && j==k) g3[IDX(i,j,k,l)] += mu;
    129:             if (k==l && i==j && !cc) (void) PetscPrintf(PETSC_COMM_WORLD,"g3[%d] += lambda;\n",IDX(i,j,k,l));
    130:             if (i==k && j==l && !cc) (void) PetscPrintf(PETSC_COMM_WORLD,"g3[%d] += mu;\n",IDX(i,j,k,l));
    131:             if (i==l && j==k && !cc) (void) PetscPrintf(PETSC_COMM_WORLD,"g3[%d] += mu;\n",IDX(i,j,k,l));
    132:           }
    133:         }
    134:       }
    135:     }
    136:   }
    137: }

    139: static void g3_uu_3d_alpha(PetscInt dim, PetscInt Nf, PetscInt NfAux,
    140:                            const PetscInt uOff[], const PetscInt uOff_x[], const PetscScalar u[], const PetscScalar u_t[], const PetscScalar u_x[],
    141:                            const PetscInt aOff[], const PetscInt aOff_x[], const PetscScalar a[], const PetscScalar a_t[], const PetscScalar a_x[],
    142:                            PetscReal t, PetscReal u_tShift, const PetscReal x[], PetscInt numConstants, const PetscScalar constants[], PetscScalar g3[])
    143: {
    144:   PetscReal mu=s_mu, lambda=s_lambda,rad;
    145:   PetscInt i;
    146:   for (i=0,rad=0.;i<dim;i++) {
    147:     PetscReal t=x[i];
    148:     rad += t*t;
    149:   }
    150:   rad = PetscSqrtReal(rad);
    151:   if (rad>0.25) {
    152:     mu *= s_soft_alpha;
    153:     lambda *= s_soft_alpha; /* we could keep the bulk the same like rubberish */
    154:   }
    155:   g3_uu_3d_private(g3,mu,lambda);
    156: }

    158: static void g3_uu_3d(PetscInt dim, PetscInt Nf, PetscInt NfAux,
    159:                      const PetscInt uOff[], const PetscInt uOff_x[], const PetscScalar u[], const PetscScalar u_t[], const PetscScalar u_x[],
    160:                      const PetscInt aOff[], const PetscInt aOff_x[], const PetscScalar a[], const PetscScalar a_t[], const PetscScalar a_x[],
    161:                      PetscReal t, PetscReal u_tShift, const PetscReal x[], PetscInt numConstants, const PetscScalar constants[], PetscScalar g3[])
    162: {
    163:   g3_uu_3d_private(g3,s_mu,s_lambda);
    164: }

    166: static void f0_u(PetscInt dim, PetscInt Nf, PetscInt NfAux,
    167:                  const PetscInt uOff[], const PetscInt uOff_x[], const PetscScalar u[], const PetscScalar u_t[], const PetscScalar u_x[],
    168:                  const PetscInt aOff[], const PetscInt aOff_x[], const PetscScalar a[], const PetscScalar a_t[], const PetscScalar a_x[],
    169:                  PetscReal t, const PetscReal x[], PetscInt numConstants, const PetscScalar constants[], PetscScalar f0[])
    170: {
    171:   const    PetscInt Ncomp = dim;
    172:   PetscInt comp;

    174:   for (comp = 0; comp < Ncomp; ++comp) f0[comp] = 0.0;
    175: }

    177: /* PI_i (x_i^4 - x_i^2) */
    178: static void f0_u_x4(PetscInt dim, PetscInt Nf, PetscInt NfAux,
    179:                     const PetscInt uOff[], const PetscInt uOff_x[], const PetscScalar u[], const PetscScalar u_t[], const PetscScalar u_x[],
    180:                     const PetscInt aOff[], const PetscInt aOff_x[], const PetscScalar a[], const PetscScalar a_t[], const PetscScalar a_x[],
    181:                     PetscReal t, const PetscReal x[], PetscInt numConstants, const PetscScalar constants[], PetscScalar f0[])
    182: {
    183:   const    PetscInt Ncomp = dim;
    184:   PetscInt comp,i;

    186:   for (comp = 0; comp < Ncomp; ++comp) {
    187:     f0[comp] = 1e5;
    188:     for (i = 0; i < Ncomp; ++i) {
    189:       f0[comp] *= /* (comp+1)* */(x[i]*x[i]*x[i]*x[i] - x[i]*x[i]); /* assumes (0,1]^D domain */
    190:     }
    191:   }
    192: }

    194: PetscErrorCode zero(PetscInt dim, PetscReal time, const PetscReal x[], PetscInt Nf, PetscScalar *u, void *ctx)
    195: {
    196:   const PetscInt Ncomp = dim;
    197:   PetscInt       comp;

    199:   for (comp = 0; comp < Ncomp; ++comp) u[comp] = 0;
    200:   return 0;
    201: }

    203: int main(int argc,char **args)
    204: {
    205:   Mat                Amat;
    206:   PetscErrorCode     ierr;
    207:   SNES               snes;
    208:   KSP                ksp;
    209:   MPI_Comm           comm;
    210:   PetscMPIInt        rank;
    211: #if defined(PETSC_USE_LOG)
    212:   PetscLogStage      stage[17];
    213: #endif
    214:   PetscBool          test_nonzero_cols = PETSC_FALSE,use_nearnullspace = PETSC_TRUE,attach_nearnullspace = PETSC_FALSE;
    215:   Vec                xx,bb;
    216:   PetscInt           iter,i,N,dim = 3,cells[3] = {1,1,1},max_conv_its,local_sizes[7],run_type = 1;
    217:   DM                 dm,distdm,basedm;
    218:   PetscBool          flg;
    219:   char               convType[256];
    220:   PetscReal          Lx,mdisp[10],err[10];
    221:   const char * const options[10] = {"-ex56_dm_refine 0",
    222:                                     "-ex56_dm_refine 1",
    223:                                     "-ex56_dm_refine 2",
    224:                                     "-ex56_dm_refine 3",
    225:                                     "-ex56_dm_refine 4",
    226:                                     "-ex56_dm_refine 5",
    227:                                     "-ex56_dm_refine 6",
    228:                                     "-ex56_dm_refine 7",
    229:                                     "-ex56_dm_refine 8",
    230:                                     "-ex56_dm_refine 9"};
    232:   PetscInitialize(&argc,&args,(char*)0,help);
    233:   comm = PETSC_COMM_WORLD;
    234:   MPI_Comm_rank(comm, &rank);
    235:   /* options */
    236:   PetscOptionsBegin(comm,NULL,"3D bilinear Q1 elasticity options","");
    237:   {
    238:     i = 3;
    239:     PetscOptionsIntArray("-cells", "Number of (flux tube) processor in each dimension", "ex56.c", cells, &i, NULL);

    241:     Lx = 1.; /* or ne for rod */
    242:     max_conv_its = 3;
    243:     PetscOptionsInt("-max_conv_its","Number of iterations in convergence study","",max_conv_its,&max_conv_its,NULL);
    245:     PetscOptionsReal("-lx","Length of domain","",Lx,&Lx,NULL);
    246:     PetscOptionsReal("-alpha","material coefficient inside circle","",s_soft_alpha,&s_soft_alpha,NULL);
    247:     PetscOptionsBool("-test_nonzero_cols","nonzero test","",test_nonzero_cols,&test_nonzero_cols,NULL);
    248:     PetscOptionsBool("-use_mat_nearnullspace","MatNearNullSpace API test","",use_nearnullspace,&use_nearnullspace,NULL);
    249:     PetscOptionsBool("-attach_mat_nearnullspace","MatNearNullSpace API test (via MatSetNearNullSpace)","",attach_nearnullspace,&attach_nearnullspace,NULL);
    250:     PetscOptionsInt("-run_type","0: twisting load on cantalever, 1: 3rd order accurate convergence test","",run_type,&run_type,NULL);
    251:   }
    252:   PetscOptionsEnd();
    253:   PetscLogStageRegister("Mesh Setup", &stage[16]);
    254:   for (iter=0 ; iter<max_conv_its ; iter++) {
    255:     char str[] = "Solve 0";
    256:     str[6] += iter;
    257:     PetscLogStageRegister(str, &stage[iter]);
    258:   }
    259:   /* create DM, Plex calls DMSetup */
    260:   PetscLogStagePush(stage[16]);
    261:   DMPlexCreateBoxMesh(comm, dim, PETSC_FALSE, cells, NULL, NULL, NULL, PETSC_TRUE, &dm);
    262:   {
    263:     DMLabel         label;
    264:     IS              is;
    265:     DMCreateLabel(dm, "boundary");
    266:     DMGetLabel(dm, "boundary", &label);
    267:     DMPlexMarkBoundaryFaces(dm, 1, label);
    268:     if (!run_type) {
    269:       DMGetStratumIS(dm, "boundary", 1,  &is);
    270:       DMCreateLabel(dm,"Faces");
    271:       if (is) {
    272:         PetscInt        d, f, Nf;
    273:         const PetscInt *faces;
    274:         PetscInt        csize;
    275:         PetscSection    cs;
    276:         Vec             coordinates ;
    277:         DM              cdm;
    278:         ISGetLocalSize(is, &Nf);
    279:         ISGetIndices(is, &faces);
    280:         DMGetCoordinatesLocal(dm, &coordinates);
    281:         DMGetCoordinateDM(dm, &cdm);
    282:         DMGetLocalSection(cdm, &cs);
    283:         /* Check for each boundary face if any component of its centroid is either 0.0 or 1.0 */
    284:         for (f = 0; f < Nf; ++f) {
    285:           PetscReal   faceCoord;
    286:           PetscInt    b,v;
    287:           PetscScalar *coords = NULL;
    288:           PetscInt    Nv;
    289:           DMPlexVecGetClosure(cdm, cs, coordinates, faces[f], &csize, &coords);
    290:           Nv   = csize/dim; /* Calculate mean coordinate vector */
    291:           for (d = 0; d < dim; ++d) {
    292:             faceCoord = 0.0;
    293:             for (v = 0; v < Nv; ++v) faceCoord += PetscRealPart(coords[v*dim+d]);
    294:             faceCoord /= Nv;
    295:             for (b = 0; b < 2; ++b) {
    296:               if (PetscAbs(faceCoord - b) < PETSC_SMALL) { /* domain have not been set yet, still [0,1]^3 */
    297:                 DMSetLabelValue(dm, "Faces", faces[f], d*2+b+1);
    298:               }
    299:             }
    300:           }
    301:           DMPlexVecRestoreClosure(cdm, cs, coordinates, faces[f], &csize, &coords);
    302:         }
    303:         ISRestoreIndices(is, &faces);
    304:       }
    305:       ISDestroy(&is);
    306:       DMGetLabel(dm, "Faces", &label);
    307:       DMPlexLabelComplete(dm, label);
    308:     }
    309:   }
    310:   {
    311:     PetscInt    dimEmbed, i;
    312:     PetscInt    nCoords;
    313:     PetscScalar *coords,bounds[] = {0,1,-.5,.5,-.5,.5,}; /* x_min,x_max,y_min,y_max */
    314:     Vec         coordinates;
    315:     bounds[1] = Lx;
    316:     if (run_type==1) {
    317:       for (i = 0; i < 2*dim; i++) bounds[i] = (i%2) ? 1 : 0;
    318:     }
    319:     DMGetCoordinatesLocal(dm,&coordinates);
    320:     DMGetCoordinateDim(dm,&dimEmbed);
    322:     VecGetLocalSize(coordinates,&nCoords);
    324:     VecGetArray(coordinates,&coords);
    325:     for (i = 0; i < nCoords; i += dimEmbed) {
    326:       PetscInt    j;
    327:       PetscScalar *coord = &coords[i];
    328:       for (j = 0; j < dimEmbed; j++) {
    329:         coord[j] = bounds[2 * j] + coord[j] * (bounds[2 * j + 1] - bounds[2 * j]);
    330:       }
    331:     }
    332:     VecRestoreArray(coordinates,&coords);
    333:     DMSetCoordinatesLocal(dm,coordinates);
    334:   }

    336:   /* convert to p4est, and distribute */

    338:   PetscOptionsBegin(comm, "", "Mesh conversion options", "DMPLEX");
    339:   PetscOptionsFList("-dm_type","Convert DMPlex to another format (should not be Plex!)","ex56.c",DMList,DMPLEX,convType,256,&flg);
    340:   PetscOptionsEnd();
    341:   if (flg) {
    342:     DM newdm;
    343:     DMConvert(dm,convType,&newdm);
    344:     if (newdm) {
    345:       const char *prefix;
    346:       PetscBool isForest;
    347:       PetscObjectGetOptionsPrefix((PetscObject)dm,&prefix);
    348:       PetscObjectSetOptionsPrefix((PetscObject)newdm,prefix);
    349:       DMIsForest(newdm,&isForest);
    351:       DMDestroy(&dm);
    352:       dm   = newdm;
    353:     } else SETERRQ(PETSC_COMM_WORLD, PETSC_ERR_USER, "Convert failed?");
    354:   } else {
    355:     PetscPartitioner part;
    356:     /* Plex Distribute mesh over processes */
    357:     DMPlexGetPartitioner(dm,&part);
    358:     PetscPartitionerSetFromOptions(part);
    359:     DMPlexDistribute(dm, 0, NULL, &distdm);
    360:     if (distdm) {
    361:       const char *prefix;
    362:       PetscObjectGetOptionsPrefix((PetscObject)dm,&prefix);
    363:       PetscObjectSetOptionsPrefix((PetscObject)distdm,prefix);
    364:       DMDestroy(&dm);
    365:       dm   = distdm;
    366:     }
    367:   }
    368:   PetscLogStagePop();
    369:   basedm = dm; dm = NULL;

    371:   for (iter=0 ; iter<max_conv_its ; iter++) {
    372:     PetscLogStagePush(stage[16]);
    373:     /* make new DM */
    374:     DMClone(basedm, &dm);
    375:     PetscObjectSetOptionsPrefix((PetscObject) dm, "ex56_");
    376:     PetscObjectSetName( (PetscObject)dm,"Mesh");
    377:     if (max_conv_its > 1) {
    378:       /* If max_conv_its == 1, then we are not doing a convergence study. */
    379:       PetscOptionsInsertString(NULL,options[iter]);
    380:     }
    381:     DMSetFromOptions(dm); /* refinement done here in Plex, p4est */
    382:     /* snes */
    383:     SNESCreate(comm, &snes);
    384:     SNESSetDM(snes, dm);
    385:     /* fem */
    386:     {
    387:       const PetscInt Ncomp = dim;
    388:       const PetscInt components[] = {0,1,2};
    389:       const PetscInt Nfid = 1, Npid = 1;
    390:       const PetscInt fid[] = {1}; /* The fixed faces (x=0) */
    391:       const PetscInt pid[] = {2}; /* The faces with loading (x=L_x) */
    392:       PetscFE        fe;
    393:       PetscDS        prob;
    394:       DMLabel        label;
    395:       DM             cdm = dm;

    397:       PetscFECreateDefault(PetscObjectComm((PetscObject) dm), dim, dim, PETSC_FALSE, NULL, PETSC_DECIDE, &fe); /* elasticity */
    398:       PetscObjectSetName((PetscObject) fe, "deformation");
    399:       /* FEM prob */
    400:       DMSetField(dm, 0, NULL, (PetscObject) fe);
    401:       DMCreateDS(dm);
    402:       DMGetDS(dm, &prob);
    403:       /* setup problem */
    404:       if (run_type==1) {
    405:         PetscDSSetJacobian(prob, 0, 0, NULL, NULL, NULL, g3_uu_3d);
    406:         PetscDSSetResidual(prob, 0, f0_u_x4, f1_u_3d);
    407:       } else {
    408:         PetscWeakForm wf;
    409:         PetscInt      bd, i;

    411:         PetscDSSetJacobian(prob, 0, 0, NULL, NULL, NULL, g3_uu_3d_alpha);
    412:         PetscDSSetResidual(prob, 0, f0_u, f1_u_3d_alpha);

    414:         DMGetLabel(dm, "Faces", &label);
    415:         DMAddBoundary(dm, DM_BC_NATURAL, "traction", label, Npid, pid, 0, Ncomp, components, NULL, NULL, NULL, &bd);
    416:         PetscDSGetBoundary(prob, bd, &wf, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL);
    417:         for (i = 0; i < Npid; ++i) PetscWeakFormSetIndexBdResidual(wf, label, pid[i], 0, 0, 0, f0_bd_u_3d, 0, f1_bd_u);
    418:       }
    419:       /* bcs */
    420:       if (run_type==1) {
    421:         PetscInt id = 1;
    422:         DMGetLabel(dm, "boundary", &label);
    423:         DMAddBoundary(dm, DM_BC_ESSENTIAL, "wall", label, 1, &id, 0, 0, NULL, (void (*)(void)) zero, NULL, NULL, NULL);
    424:       } else {
    425:         DMGetLabel(dm, "Faces", &label);
    426:         DMAddBoundary(dm, DM_BC_ESSENTIAL, "fixed", label, Nfid, fid, 0, Ncomp, components, (void (*)(void)) zero, NULL, NULL, NULL);
    427:       }
    428:       while (cdm) {
    429:         DMCopyDisc(dm, cdm);
    430:         DMGetCoarseDM(cdm, &cdm);
    431:       }
    432:       PetscFEDestroy(&fe);
    433:     }
    434:     /* vecs & mat */
    435:     DMCreateGlobalVector(dm,&xx);
    436:     VecDuplicate(xx, &bb);
    437:     PetscObjectSetName((PetscObject) bb, "b");
    438:     PetscObjectSetName((PetscObject) xx, "u");
    439:     DMCreateMatrix(dm, &Amat);
    440:     MatSetOption(Amat,MAT_SYMMETRIC,PETSC_TRUE);        /* Some matrix kernels can take advantage of symmetry if we set this. */
    441:     MatSetOption(Amat,MAT_SYMMETRY_ETERNAL,PETSC_TRUE); /* Inform PETSc that Amat is always symmetric, so info set above isn't lost. */
    442:     MatSetBlockSize(Amat,3);
    443:     MatSetOption(Amat,MAT_SPD,PETSC_TRUE);
    444:     VecGetSize(bb,&N);
    445:     local_sizes[iter] = N;
    446:     PetscInfo(snes,"%D global equations, %D vertices\n",N,N/dim);
    447:     if ((use_nearnullspace || attach_nearnullspace) && N/dim > 1) {
    448:       /* Set up the near null space (a.k.a. rigid body modes) that will be used by the multigrid preconditioner */
    449:       DM           subdm;
    450:       MatNullSpace nearNullSpace;
    451:       PetscInt     fields = 0;
    452:       PetscObject  deformation;
    453:       DMCreateSubDM(dm, 1, &fields, NULL, &subdm);
    454:       DMPlexCreateRigidBody(subdm, 0, &nearNullSpace);
    455:       DMGetField(dm, 0, NULL, &deformation);
    456:       PetscObjectCompose(deformation, "nearnullspace", (PetscObject) nearNullSpace);
    457:       DMDestroy(&subdm);
    458:       if (attach_nearnullspace) {
    459:         MatSetNearNullSpace(Amat,nearNullSpace);
    460:       }
    461:       MatNullSpaceDestroy(&nearNullSpace); /* created by DM and destroyed by Mat */
    462:     }
    463:     DMPlexSetSNESLocalFEM(dm,NULL,NULL,NULL);
    464:     SNESSetJacobian(snes, Amat, Amat, NULL, NULL);
    465:     SNESSetFromOptions(snes);
    466:     DMSetUp(dm);
    467:     PetscLogStagePop();
    468:     PetscLogStagePush(stage[16]);
    469:     /* ksp */
    470:     SNESGetKSP(snes, &ksp);
    471:     KSPSetComputeSingularValues(ksp,PETSC_TRUE);
    472:     /* test BCs */
    473:     VecZeroEntries(xx);
    474:     if (test_nonzero_cols) {
    475:       if (rank == 0) {
    476:         VecSetValue(xx,0,1.0,INSERT_VALUES);
    477:       }
    478:       VecAssemblyBegin(xx);
    479:       VecAssemblyEnd(xx);
    480:     }
    481:     VecZeroEntries(bb);
    482:     VecGetSize(bb,&i);
    483:     local_sizes[iter] = i;
    484:     PetscInfo(snes,"%D equations in vector, %D vertices\n",i,i/dim);
    485:     PetscLogStagePop();
    486:     /* solve */
    487:     PetscLogStagePush(stage[iter]);
    488:     SNESSolve(snes, bb, xx);
    489:     PetscLogStagePop();
    490:     VecNorm(xx,NORM_INFINITY,&mdisp[iter]);
    491:     DMViewFromOptions(dm, NULL, "-dm_view");
    492:     {
    493:       PetscViewer       viewer = NULL;
    494:       PetscViewerFormat fmt;
    495:       PetscOptionsGetViewer(comm,NULL,"ex56_","-vec_view",&viewer,&fmt,&flg);
    496:       if (flg) {
    497:         PetscViewerPushFormat(viewer,fmt);
    498:         VecView(xx,viewer);
    499:         VecView(bb,viewer);
    500:         PetscViewerPopFormat(viewer);
    501:       }
    502:       PetscViewerDestroy(&viewer);
    503:     }
    504:     /* Free work space */
    505:     DMDestroy(&dm);
    506:     SNESDestroy(&snes);
    507:     VecDestroy(&xx);
    508:     VecDestroy(&bb);
    509:     MatDestroy(&Amat);
    510:   }
    511:   DMDestroy(&basedm);
    512:   if (run_type==1) err[0] = 59.975208 - mdisp[0]; /* error with what I think is the exact solution */
    513:   else             err[0] = 171.038 - mdisp[0];
    514:   for (iter=1 ; iter<max_conv_its ; iter++) {
    515:     if (run_type==1) err[iter] = 59.975208 - mdisp[iter];
    516:     else             err[iter] = 171.038 - mdisp[iter];
    517:     PetscPrintf(PETSC_COMM_WORLD,"[%d] %D) N=%12D, max displ=%9.7e, disp diff=%9.2e, error=%4.3e, rate=%3.2g\n",rank,iter,local_sizes[iter],(double)mdisp[iter],
    518:                        (double)(mdisp[iter]-mdisp[iter-1]),(double)err[iter],(double)(PetscLogReal(err[iter-1]/err[iter])/PetscLogReal(2.)));
    519:   }

    521:   PetscFinalize();
    522:   return 0;
    523: }

    525: /*TEST

    527:   test:
    528:     suffix: 0
    529:     nsize: 4
    530:     requires: !single
    531:     args: -cells 2,2,1 -max_conv_its 2 -petscspace_degree 2 -snes_max_it 2 -ksp_max_it 100 -ksp_type cg -ksp_rtol 1.e-10 -ksp_norm_type unpreconditioned -snes_rtol 1.e-10 -pc_type gamg -pc_gamg_esteig_ksp_max_it 10 -pc_gamg_type agg -pc_gamg_agg_nsmooths 1 -pc_gamg_coarse_eq_limit 10 -pc_gamg_reuse_interpolation true -pc_gamg_square_graph 1 -pc_gamg_threshold 0.05 -pc_gamg_threshold_scale .0 -ksp_converged_reason -snes_monitor_short -ksp_monitor_short -snes_converged_reason -use_mat_nearnullspace true -mg_levels_ksp_max_it 2 -mg_levels_ksp_type chebyshev -mg_levels_ksp_chebyshev_esteig 0,0.05,0,1.1 -mg_levels_pc_type jacobi -petscpartitioner_type simple -matptap_via scalable -ex56_dm_view
    532:     timeoutfactor: 2

    534:   # HYPRE PtAP broken with complex numbers
    535:   test:
    536:     suffix: hypre
    537:     requires: hypre !single !complex !defined(PETSC_HAVE_HYPRE_DEVICE)
    538:     nsize: 4
    539:     args: -cells 2,2,1 -max_conv_its 2 -lx 1. -alpha .01 -petscspace_degree 2 -ksp_type cg -ksp_monitor_short -ksp_rtol 1.e-8 -pc_type hypre -pc_hypre_type boomeramg -pc_hypre_boomeramg_no_CF true -pc_hypre_boomeramg_agg_nl 1 -pc_hypre_boomeramg_coarsen_type HMIS -pc_hypre_boomeramg_interp_type ext+i -ksp_converged_reason -use_mat_nearnullspace true -petscpartitioner_type simple

    541:   test:
    542:     suffix: ml
    543:     requires: ml !single
    544:     nsize: 4
    545:     args: -cells 2,2,1 -max_conv_its 2 -lx 1. -alpha .01 -petscspace_degree 2 -ksp_type cg -ksp_monitor_short -ksp_converged_reason -ksp_rtol 1.e-8 -pc_type ml -mg_levels_ksp_type chebyshev -mg_levels_ksp_max_it 3 -mg_levels_ksp_chebyshev_esteig 0,0.05,0,1.05 -mg_levels_pc_type sor -petscpartitioner_type simple -use_mat_nearnullspace

    547:   test:
    548:     suffix: hpddm
    549:     requires: hpddm slepc !single defined(PETSC_HAVE_DYNAMIC_LIBRARIES) defined(PETSC_USE_SHARED_LIBRARIES)
    550:     nsize: 4
    551:     args: -cells 2,2,1 -max_conv_its 2 -lx 1. -alpha .01 -petscspace_degree 2 -ksp_type fgmres -ksp_monitor_short -ksp_converged_reason -ksp_rtol 1.e-8 -pc_type hpddm -petscpartitioner_type simple -pc_hpddm_levels_1_sub_pc_type lu -pc_hpddm_levels_1_eps_nev 6 -pc_hpddm_coarse_p 1 -pc_hpddm_coarse_pc_type svd

    553:   test:
    554:     suffix: repart
    555:     nsize: 4
    556:     requires: parmetis !single
    557:     args: -cells 8,2,2 -max_conv_its 1 -petscspace_degree 2 -snes_max_it 4 -ksp_max_it 100 -ksp_type cg -ksp_rtol 1.e-2 -ksp_norm_type unpreconditioned -snes_rtol 1.e-3 -pc_type gamg -pc_gamg_esteig_ksp_max_it 10 -pc_gamg_type agg -pc_gamg_agg_nsmooths 1 -pc_gamg_square_graph 1 -pc_gamg_threshold 0.05 -pc_gamg_threshold_scale .0 -use_mat_nearnullspace true -mg_levels_ksp_max_it 2 -mg_levels_ksp_type chebyshev -mg_levels_ksp_chebyshev_esteig 0,0.05,0,1.05 -mg_levels_pc_type jacobi -pc_gamg_mat_partitioning_type parmetis -pc_gamg_repartition true -snes_converged_reason -pc_gamg_process_eq_limit 20 -pc_gamg_coarse_eq_limit 10 -ksp_converged_reason -snes_converged_reason -pc_gamg_reuse_interpolation true

    559:   test:
    560:     suffix: bddc
    561:     nsize: 4
    562:     requires: !single
    563:     args: -cells 2,2,1 -max_conv_its 2 -lx 1. -alpha .01 -petscspace_degree 2 -ksp_type cg -ksp_monitor_short -ksp_rtol 1.e-8 -ksp_converged_reason -petscpartitioner_type simple -ex56_dm_mat_type is -matis_localmat_type {{sbaij baij aij}} -pc_type bddc

    565:   testset:
    566:     nsize: 4
    567:     requires: !single
    568:     args: -cells 2,2,1 -max_conv_its 2 -lx 1. -alpha .01 -petscspace_degree 2 -ksp_type cg -ksp_monitor_short -ksp_rtol 1.e-10 -ksp_converged_reason -petscpartitioner_type simple -ex56_dm_mat_type is -matis_localmat_type aij -pc_type bddc -attach_mat_nearnullspace {{0 1}separate output}
    569:     test:
    570:       suffix: bddc_approx_gamg
    571:       args: -pc_bddc_switch_static -prefix_push pc_bddc_dirichlet_ -approximate -pc_type gamg -pc_gamg_esteig_ksp_max_it 10 -pc_gamg_type agg -pc_gamg_agg_nsmooths 1 -pc_gamg_reuse_interpolation true -pc_gamg_square_graph 1 -pc_gamg_threshold 0.05 -pc_gamg_threshold_scale .0 -mg_levels_ksp_max_it 1 -mg_levels_ksp_type chebyshev -prefix_pop -prefix_push pc_bddc_neumann_ -approximate -pc_type gamg -pc_gamg_esteig_ksp_max_it 10 -pc_gamg_type agg -pc_gamg_agg_nsmooths 1 -pc_gamg_coarse_eq_limit 10 -pc_gamg_reuse_interpolation true -pc_gamg_square_graph 1 -pc_gamg_threshold 0.05 -pc_gamg_threshold_scale .0 -mg_levels_ksp_max_it 1 -mg_levels_ksp_type chebyshev -prefix_pop
    572:     # HYPRE PtAP broken with complex numbers
    573:     test:
    574:       requires: hypre !complex !defined(PETSC_HAVE_HYPRE_DEVICE)
    575:       suffix: bddc_approx_hypre
    576:       args: -pc_bddc_switch_static -prefix_push pc_bddc_dirichlet_ -pc_type hypre -pc_hypre_boomeramg_no_CF true -pc_hypre_boomeramg_strong_threshold 0.75 -pc_hypre_boomeramg_agg_nl 1 -pc_hypre_boomeramg_coarsen_type HMIS -pc_hypre_boomeramg_interp_type ext+i -prefix_pop -prefix_push pc_bddc_neumann_ -pc_type hypre -pc_hypre_boomeramg_no_CF true -pc_hypre_boomeramg_strong_threshold 0.75 -pc_hypre_boomeramg_agg_nl 1 -pc_hypre_boomeramg_coarsen_type HMIS -pc_hypre_boomeramg_interp_type ext+i -prefix_pop
    577:     test:
    578:       requires: ml
    579:       suffix: bddc_approx_ml
    580:       args: -pc_bddc_switch_static -prefix_push pc_bddc_dirichlet_ -approximate -pc_type ml -mg_levels_ksp_max_it 1 -mg_levels_ksp_type chebyshev -prefix_pop -prefix_push pc_bddc_neumann_ -approximate -pc_type ml -mg_levels_ksp_max_it 1 -mg_levels_ksp_type chebyshev -prefix_pop

    582:   test:
    583:     suffix: fetidp
    584:     nsize: 4
    585:     requires: !single
    586:     args: -cells 2,2,1 -max_conv_its 2 -lx 1. -alpha .01 -petscspace_degree 2 -ksp_type fetidp -fetidp_ksp_type cg -ksp_monitor_short -ksp_rtol 1.e-8 -ksp_converged_reason -petscpartitioner_type simple -ex56_dm_mat_type is -matis_localmat_type {{sbaij baij aij}}

    588:   test:
    589:     suffix: bddc_elast
    590:     nsize: 4
    591:     requires: !single
    592:     args: -cells 2,2,1 -max_conv_its 2 -lx 1. -alpha .01 -petscspace_degree 2 -ksp_type cg -ksp_monitor_short -ksp_rtol 1.e-8 -ksp_converged_reason -petscpartitioner_type simple -ex56_dm_mat_type is -matis_localmat_type sbaij -pc_type bddc -pc_bddc_monolithic -attach_mat_nearnullspace

    594:   test:
    595:     suffix: fetidp_elast
    596:     nsize: 4
    597:     requires: !single
    598:     args: -cells 2,2,1 -max_conv_its 2 -lx 1. -alpha .01 -petscspace_degree 2 -ksp_type fetidp -fetidp_ksp_type cg -ksp_monitor_short -ksp_rtol 1.e-8 -ksp_converged_reason -petscpartitioner_type simple -ex56_dm_mat_type is -matis_localmat_type sbaij -fetidp_bddc_pc_bddc_monolithic -attach_mat_nearnullspace

    600:   testset:
    601:     nsize: 4
    602:     requires: !single
    603:     args: -cells 2,2,1 -max_conv_its 2 -petscspace_degree 2 -snes_max_it 2 -ksp_max_it 100 -ksp_type cg -ksp_rtol 1.e-10 -ksp_norm_type unpreconditioned -snes_rtol 1.e-10 -pc_type gamg -pc_gamg_esteig_ksp_max_it 10 -pc_gamg_type agg -pc_gamg_agg_nsmooths 1 -pc_gamg_coarse_eq_limit 10 -pc_gamg_reuse_interpolation true -pc_gamg_square_graph 1 -pc_gamg_threshold 0.05 -pc_gamg_threshold_scale .0 -use_mat_nearnullspace true -mg_levels_ksp_max_it 2 -mg_levels_ksp_type chebyshev -mg_levels_ksp_chebyshev_esteig 0,0.05,0,1.05 -mg_levels_pc_type jacobi -ksp_monitor_short -ksp_converged_reason -snes_converged_reason -snes_monitor_short -ex56_dm_view -petscpartitioner_type simple -pc_gamg_process_eq_limit 20
    604:     output_file: output/ex56_cuda.out

    606:     test:
    607:       suffix: cuda
    608:       requires: cuda
    609:       args: -ex56_dm_mat_type aijcusparse -ex56_dm_vec_type cuda

    611:     test:
    612:       suffix: viennacl
    613:       requires: viennacl
    614:       args: -ex56_dm_mat_type aijviennacl -ex56_dm_vec_type viennacl

    616:     test:
    617:       suffix: kokkos
    618:       requires: !sycl kokkos_kernels
    619:       args: -ex56_dm_mat_type aijkokkos -ex56_dm_vec_type kokkos
    620:   # Don't run AIJMKL caes with complex scalars because of convergence issues.
    621:   # Note that we need to test both single and multiple MPI rank cases, because these use different sparse MKL routines to implement the PtAP operation.
    622:   test:
    623:     suffix: seqaijmkl
    624:     nsize: 1
    625:     requires: defined(PETSC_HAVE_MKL_SPARSE_OPTIMIZE) !single !complex
    626:     args: -cells 2,2,1 -max_conv_its 2 -petscspace_degree 2 -snes_max_it 2 -ksp_max_it 100 -ksp_type cg -ksp_rtol 1.e-11 -ksp_norm_type unpreconditioned -snes_rtol 1.e-10 -pc_type gamg -pc_gamg_type agg -pc_gamg_agg_nsmooths 1 -pc_gamg_coarse_eq_limit 1000 -pc_gamg_reuse_interpolation true -pc_gamg_square_graph 1 -pc_gamg_threshold 0.05 -pc_gamg_threshold_scale .0 -ksp_converged_reason -snes_monitor_short -ksp_monitor_short -snes_converged_reason -use_mat_nearnullspace true -mg_levels_ksp_max_it 1 -mg_levels_ksp_type chebyshev -pc_gamg_esteig_ksp_type cg -pc_gamg_esteig_ksp_max_it 10 -mg_levels_ksp_chebyshev_esteig 0,0.05,0,1.1 -mg_levels_pc_type jacobi -petscpartitioner_type simple -mat_block_size 3 -ex56_dm_view -run_type 1 -mat_seqaij_type seqaijmkl
    627:     timeoutfactor: 2

    629:   test:
    630:     suffix: mpiaijmkl
    631:     nsize: 2
    632:     requires: defined(PETSC_HAVE_MKL_SPARSE_OPTIMIZE) !single !complex
    633:     args: -cells 2,2,1 -max_conv_its 2 -petscspace_degree 2 -snes_max_it 2 -ksp_max_it 100 -ksp_type cg -ksp_rtol 1.e-11 -ksp_norm_type unpreconditioned -snes_rtol 1.e-10 -pc_type gamg -pc_gamg_type agg -pc_gamg_agg_nsmooths 1 -pc_gamg_coarse_eq_limit 1000 -pc_gamg_reuse_interpolation true -pc_gamg_square_graph 1 -pc_gamg_threshold 0.05 -pc_gamg_threshold_scale .0 -ksp_converged_reason -snes_monitor_short -ksp_monitor_short -snes_converged_reason -use_mat_nearnullspace true -mg_levels_ksp_max_it 1 -mg_levels_ksp_type chebyshev -pc_gamg_esteig_ksp_type cg -pc_gamg_esteig_ksp_max_it 10 -mg_levels_ksp_chebyshev_esteig 0,0.05,0,1.1 -mg_levels_pc_type jacobi -petscpartitioner_type simple -mat_block_size 3 -ex56_dm_view -run_type 1 -mat_seqaij_type seqaijmkl
    634:     timeoutfactor: 2

    636: TEST*/